Note: This is test shard 4 of 8.
[==========] Running 7 tests from 6 test suites.
[----------] Global test environment set-up.
[----------] 1 test from LocationAwareRebalancingBasicTest
[ RUN ] LocationAwareRebalancingBasicTest.Basic
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20240502 01:58:12.057916 8053 test_util.cc:268] Using random seed: -277014742
I20240502 01:58:12.064512 8053 ts_itest-base.cc:112] Starting cluster with:
I20240502 01:58:12.064574 8053 ts_itest-base.cc:113] --------------
I20240502 01:58:12.064595 8053 ts_itest-base.cc:114] 6 tablet servers
I20240502 01:58:12.064613 8053 ts_itest-base.cc:115] 3 replicas per TS
I20240502 01:58:12.064628 8053 ts_itest-base.cc:116] --------------
2024-05-02T01:58:12Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T01:58:12Z Disabled control of system clock
I20240502 01:58:12.074026 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.7.221.126:39109
--webserver_interface=127.7.221.126
--webserver_port=0
--builtin_ntp_servers=127.7.221.84:35093
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.7.221.126:39109
--location_mapping_cmd=/tmp/dist-test-tasko3zIVx/build/debug/bin/testdata/assign-location.py --state_store=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/location-assignment.state --map /A:2 --map /B:2 --map /C:2
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false with env {}
W20240502 01:58:12.183864 8065 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:12.184104 8065 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:12.184144 8065 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:12.187188 8065 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:12.187254 8065 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 01:58:12.187276 8065 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:12.187300 8065 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 01:58:12.187317 8065 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 01:58:12.190987 8065 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:35093
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--location_mapping_cmd=/tmp/dist-test-tasko3zIVx/build/debug/bin/testdata/assign-location.py --state_store=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/location-assignment.state --map /A:2 --map /B:2 --map /C:2
--ipki_ca_key_size=768
--master_addresses=127.7.221.126:39109
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.7.221.126:39109
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.7.221.126
--webserver_port=0
--never_fsync=true
--heap_profile_path=/tmp/kudu.8065
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:12.191851 8065 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:12.193011 8065 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:12.200951 8073 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:12.201021 8070 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:12.201203 8071 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:12.202129 8065 server_base.cc:1034] running on GCE node
I20240502 01:58:12.202647 8065 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:12.203593 8065 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:12.204749 8065 hybrid_clock.cc:648] HybridClock initialized: now 1714615092204742 us; error 33 us; skew 500 ppm
I20240502 01:58:12.206781 8065 webserver.cc:460] Webserver started at http://127.7.221.126:36461/ using document root <none> and password file <none>
I20240502 01:58:12.207372 8065 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:12.207430 8065 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:12.207626 8065 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:12.209342 8065 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "8671b1c70fa1447987c2e9c020c526d6"
format_stamp: "Formatted at 2024-05-02 01:58:12 on dist-test-slave-dmxt"
I20240502 01:58:12.209817 8065 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "8671b1c70fa1447987c2e9c020c526d6"
format_stamp: "Formatted at 2024-05-02 01:58:12 on dist-test-slave-dmxt"
I20240502 01:58:12.213476 8065 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.004s
I20240502 01:58:12.215855 8079 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:12.216886 8065 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:12.217018 8065 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
uuid: "8671b1c70fa1447987c2e9c020c526d6"
format_stamp: "Formatted at 2024-05-02 01:58:12 on dist-test-slave-dmxt"
I20240502 01:58:12.217130 8065 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:12.237008 8065 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:12.237648 8065 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:12.237813 8065 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:12.244180 8065 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.126:39109
I20240502 01:58:12.244189 8132 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.126:39109 every 8 connection(s)
I20240502 01:58:12.245178 8065 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 01:58:12.247785 8133 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:12.250015 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 8065
I20240502 01:58:12.250173 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 01:58:12.254264 8133 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6: Bootstrap starting.
I20240502 01:58:12.257174 8133 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:12.258054 8133 log.cc:824] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:12.260135 8133 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6: No bootstrap required, opened a new log
I20240502 01:58:12.263535 8133 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8671b1c70fa1447987c2e9c020c526d6" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 39109 } }
I20240502 01:58:12.263785 8133 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:12.263868 8133 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8671b1c70fa1447987c2e9c020c526d6, State: Initialized, Role: FOLLOWER
I20240502 01:58:12.264361 8133 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8671b1c70fa1447987c2e9c020c526d6" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 39109 } }
I20240502 01:58:12.264480 8133 raft_consensus.cc:388] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 01:58:12.264539 8133 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 01:58:12.264690 8133 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:12.265615 8133 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8671b1c70fa1447987c2e9c020c526d6" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 39109 } }
I20240502 01:58:12.265956 8133 leader_election.cc:302] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 8671b1c70fa1447987c2e9c020c526d6; no voters:
I20240502 01:58:12.266252 8133 leader_election.cc:288] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 01:58:12.266348 8138 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:12.266510 8138 raft_consensus.cc:686] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [term 1 LEADER]: Becoming Leader. State: Replica: 8671b1c70fa1447987c2e9c020c526d6, State: Running, Role: LEADER
I20240502 01:58:12.266870 8138 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8671b1c70fa1447987c2e9c020c526d6" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 39109 } }
I20240502 01:58:12.267500 8133 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [sys.catalog]: configured and running, proceeding with master startup.
I20240502 01:58:12.269008 8139 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8671b1c70fa1447987c2e9c020c526d6. Latest consensus state: current_term: 1 leader_uuid: "8671b1c70fa1447987c2e9c020c526d6" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8671b1c70fa1447987c2e9c020c526d6" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 39109 } } }
I20240502 01:58:12.269125 8139 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [sys.catalog]: This master's current role is: LEADER
I20240502 01:58:12.269068 8140 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "8671b1c70fa1447987c2e9c020c526d6" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8671b1c70fa1447987c2e9c020c526d6" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 39109 } } }
I20240502 01:58:12.269201 8140 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6 [sys.catalog]: This master's current role is: LEADER
I20240502 01:58:12.269845 8142 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 01:58:12.272488 8142 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 01:58:12.277487 8142 catalog_manager.cc:1347] Generated new cluster ID: d2fc9ec2a01b40e5b05addde982eba4e
I20240502 01:58:12.277554 8142 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 01:58:12.308262 8142 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 01:58:12.309229 8142 catalog_manager.cc:1504] Loading token signing keys...
I20240502 01:58:12.314047 8142 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 8671b1c70fa1447987c2e9c020c526d6: Generated new TSK 0
I20240502 01:58:12.314814 8142 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 01:58:12.324108 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.65:0
--local_ip_for_outbound_sockets=127.7.221.65
--webserver_interface=127.7.221.65
--webserver_port=0
--tserver_master_addrs=127.7.221.126:39109
--builtin_ntp_servers=127.7.221.84:35093
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 01:58:12.432545 8157 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:12.432777 8157 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:12.432854 8157 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:12.435822 8157 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:12.435902 8157 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:12.435997 8157 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.65
I20240502 01:58:12.439522 8157 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:35093
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.7.221.65
--webserver_port=0
--tserver_master_addrs=127.7.221.126:39109
--never_fsync=true
--heap_profile_path=/tmp/kudu.8157
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.65
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:12.440464 8157 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:12.441629 8157 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:12.449669 8162 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:12.449726 8163 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:12.450538 8165 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:12.451445 8157 server_base.cc:1034] running on GCE node
I20240502 01:58:12.451864 8157 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:12.452490 8157 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:12.453652 8157 hybrid_clock.cc:648] HybridClock initialized: now 1714615092453639 us; error 29 us; skew 500 ppm
I20240502 01:58:12.455651 8157 webserver.cc:460] Webserver started at http://127.7.221.65:45975/ using document root <none> and password file <none>
I20240502 01:58:12.456223 8157 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:12.456288 8157 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:12.456470 8157 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:12.458222 8157 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
format_stamp: "Formatted at 2024-05-02 01:58:12 on dist-test-slave-dmxt"
I20240502 01:58:12.458747 8157 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
format_stamp: "Formatted at 2024-05-02 01:58:12 on dist-test-slave-dmxt"
I20240502 01:58:12.462096 8157 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:12.464285 8171 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:12.465301 8157 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:12.465417 8157 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
format_stamp: "Formatted at 2024-05-02 01:58:12 on dist-test-slave-dmxt"
I20240502 01:58:12.465499 8157 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:12.485853 8157 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:12.486527 8157 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:12.486729 8157 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:12.487299 8157 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:12.488190 8157 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:12.488240 8157 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:12.488298 8157 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:12.488332 8157 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:12.496800 8157 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.65:41919
I20240502 01:58:12.496829 8285 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.65:41919 every 8 connection(s)
I20240502 01:58:12.497798 8157 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 01:58:12.500406 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 8157
I20240502 01:58:12.500515 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 01:58:12.502755 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.66:0
--local_ip_for_outbound_sockets=127.7.221.66
--webserver_interface=127.7.221.66
--webserver_port=0
--tserver_master_addrs=127.7.221.126:39109
--builtin_ntp_servers=127.7.221.84:35093
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:12.507220 8286 heartbeater.cc:347] Connected to a master server at 127.7.221.126:39109
I20240502 01:58:12.507460 8286 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:12.507992 8286 heartbeater.cc:511] Master 127.7.221.126:39109 requested a full tablet report, sending...
I20240502 01:58:12.547307 8097 ts_manager.cc:194] Registered new tserver with Master: e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919)
I20240502 01:58:12.548688 8097 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.65:60827
W20240502 01:58:12.617832 8289 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:12.618098 8289 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:12.618170 8289 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:12.621390 8289 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:12.621455 8289 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:12.621548 8289 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.66
I20240502 01:58:12.625202 8289 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:35093
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.7.221.66
--webserver_port=0
--tserver_master_addrs=127.7.221.126:39109
--never_fsync=true
--heap_profile_path=/tmp/kudu.8289
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.66
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:12.626081 8289 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:12.627274 8289 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:12.634873 8296 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:12.635100 8298 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:12.635792 8295 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:12.636643 8289 server_base.cc:1034] running on GCE node
I20240502 01:58:12.636966 8289 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:12.637460 8289 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:12.638653 8289 hybrid_clock.cc:648] HybridClock initialized: now 1714615092638623 us; error 38 us; skew 500 ppm
I20240502 01:58:12.640445 8289 webserver.cc:460] Webserver started at http://127.7.221.66:42353/ using document root <none> and password file <none>
I20240502 01:58:12.640971 8289 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:12.641023 8289 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:12.641178 8289 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:12.642797 8289 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "73ee70f488334b8f8ffb83aef6456e15"
format_stamp: "Formatted at 2024-05-02 01:58:12 on dist-test-slave-dmxt"
I20240502 01:58:12.643237 8289 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "73ee70f488334b8f8ffb83aef6456e15"
format_stamp: "Formatted at 2024-05-02 01:58:12 on dist-test-slave-dmxt"
I20240502 01:58:12.646366 8289 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 01:58:12.648638 8304 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:12.649617 8289 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:12.649714 8289 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "73ee70f488334b8f8ffb83aef6456e15"
format_stamp: "Formatted at 2024-05-02 01:58:12 on dist-test-slave-dmxt"
I20240502 01:58:12.649787 8289 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:12.690436 8289 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:12.691190 8289 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:12.691340 8289 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:12.691872 8289 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:12.692831 8289 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:12.692881 8289 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:12.692924 8289 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:12.692943 8289 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:12.702042 8289 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.66:37503
I20240502 01:58:12.702071 8418 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.66:37503 every 8 connection(s)
I20240502 01:58:12.703078 8289 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 01:58:12.709766 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 8289
I20240502 01:58:12.709877 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 01:58:12.712219 8419 heartbeater.cc:347] Connected to a master server at 127.7.221.126:39109
I20240502 01:58:12.712314 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.67:0
--local_ip_for_outbound_sockets=127.7.221.67
--webserver_interface=127.7.221.67
--webserver_port=0
--tserver_master_addrs=127.7.221.126:39109
--builtin_ntp_servers=127.7.221.84:35093
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:12.712517 8419 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:12.713054 8419 heartbeater.cc:511] Master 127.7.221.126:39109 requested a full tablet report, sending...
I20240502 01:58:12.761214 8097 ts_manager.cc:194] Registered new tserver with Master: 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503)
I20240502 01:58:12.761920 8097 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.66:34701
W20240502 01:58:12.838809 8422 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:12.839013 8422 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:12.839079 8422 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:12.842120 8422 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:12.842193 8422 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:12.842285 8422 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.67
I20240502 01:58:12.845888 8422 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:35093
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.7.221.67
--webserver_port=0
--tserver_master_addrs=127.7.221.126:39109
--never_fsync=true
--heap_profile_path=/tmp/kudu.8422
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.67
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:12.846776 8422 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:12.847862 8422 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:12.855919 8431 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:12.856112 8429 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:12.856755 8428 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:12.857441 8422 server_base.cc:1034] running on GCE node
I20240502 01:58:12.857854 8422 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:12.858466 8422 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:12.859623 8422 hybrid_clock.cc:648] HybridClock initialized: now 1714615092859599 us; error 40 us; skew 500 ppm
I20240502 01:58:12.861665 8422 webserver.cc:460] Webserver started at http://127.7.221.67:41293/ using document root <none> and password file <none>
I20240502 01:58:12.862246 8422 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:12.862323 8422 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:12.862565 8422 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:12.864307 8422 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "e325acceaed541869d43c1f762530c2e"
format_stamp: "Formatted at 2024-05-02 01:58:12 on dist-test-slave-dmxt"
I20240502 01:58:12.864811 8422 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "e325acceaed541869d43c1f762530c2e"
format_stamp: "Formatted at 2024-05-02 01:58:12 on dist-test-slave-dmxt"
I20240502 01:58:12.868229 8422 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 01:58:12.870429 8438 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:12.871517 8422 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:12.871618 8422 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "e325acceaed541869d43c1f762530c2e"
format_stamp: "Formatted at 2024-05-02 01:58:12 on dist-test-slave-dmxt"
I20240502 01:58:12.871708 8422 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:12.909409 8422 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:12.910166 8422 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:12.910336 8422 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:12.910943 8422 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:12.911903 8422 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:12.911959 8422 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:12.912014 8422 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:12.912045 8422 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:12.920549 8422 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.67:42267
I20240502 01:58:12.920678 8553 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.67:42267 every 8 connection(s)
I20240502 01:58:12.921595 8422 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 01:58:12.929857 8554 heartbeater.cc:347] Connected to a master server at 127.7.221.126:39109
I20240502 01:58:12.930058 8554 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:12.930505 8554 heartbeater.cc:511] Master 127.7.221.126:39109 requested a full tablet report, sending...
I20240502 01:58:12.930963 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 8422
I20240502 01:58:12.931067 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 01:58:12.935365 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.68:0
--local_ip_for_outbound_sockets=127.7.221.68
--webserver_interface=127.7.221.68
--webserver_port=0
--tserver_master_addrs=127.7.221.126:39109
--builtin_ntp_servers=127.7.221.84:35093
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:12.964293 8097 ts_manager.cc:194] Registered new tserver with Master: e325acceaed541869d43c1f762530c2e (127.7.221.67:42267)
I20240502 01:58:12.965236 8097 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.67:59255
W20240502 01:58:13.043484 8558 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:13.043710 8558 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:13.043758 8558 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:13.046680 8558 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:13.046738 8558 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:13.046800 8558 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.68
I20240502 01:58:13.050159 8558 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:35093
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.68:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.7.221.68
--webserver_port=0
--tserver_master_addrs=127.7.221.126:39109
--never_fsync=true
--heap_profile_path=/tmp/kudu.8558
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.68
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:13.051005 8558 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:13.052057 8558 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:13.059422 8564 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:13.059648 8566 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:13.059918 8563 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:13.060674 8558 server_base.cc:1034] running on GCE node
I20240502 01:58:13.061014 8558 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:13.061498 8558 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:13.062649 8558 hybrid_clock.cc:648] HybridClock initialized: now 1714615093062623 us; error 36 us; skew 500 ppm
I20240502 01:58:13.064478 8558 webserver.cc:460] Webserver started at http://127.7.221.68:37623/ using document root <none> and password file <none>
I20240502 01:58:13.064998 8558 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:13.065052 8558 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:13.065227 8558 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:13.066854 8558 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "ef9a99ad58b84fd2ada7f035f27361b1"
format_stamp: "Formatted at 2024-05-02 01:58:13 on dist-test-slave-dmxt"
I20240502 01:58:13.067301 8558 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "ef9a99ad58b84fd2ada7f035f27361b1"
format_stamp: "Formatted at 2024-05-02 01:58:13 on dist-test-slave-dmxt"
I20240502 01:58:13.070482 8558 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.001s sys 0.002s
I20240502 01:58:13.072610 8572 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:13.073604 8558 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:13.073704 8558 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "ef9a99ad58b84fd2ada7f035f27361b1"
format_stamp: "Formatted at 2024-05-02 01:58:13 on dist-test-slave-dmxt"
I20240502 01:58:13.073798 8558 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:13.091904 8558 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:13.092592 8558 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:13.092756 8558 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:13.093308 8558 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:13.094245 8558 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:13.094295 8558 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:13.094342 8558 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:13.094372 8558 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:13.103313 8558 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.68:38021
I20240502 01:58:13.103341 8688 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.68:38021 every 8 connection(s)
I20240502 01:58:13.104377 8558 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 01:58:13.111033 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 8558
I20240502 01:58:13.111177 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 01:58:13.113112 8689 heartbeater.cc:347] Connected to a master server at 127.7.221.126:39109
I20240502 01:58:13.113366 8689 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:13.113452 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.69:0
--local_ip_for_outbound_sockets=127.7.221.69
--webserver_interface=127.7.221.69
--webserver_port=0
--tserver_master_addrs=127.7.221.126:39109
--builtin_ntp_servers=127.7.221.84:35093
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:13.114027 8689 heartbeater.cc:511] Master 127.7.221.126:39109 requested a full tablet report, sending...
I20240502 01:58:13.162356 8096 ts_manager.cc:194] Registered new tserver with Master: ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021)
I20240502 01:58:13.163168 8096 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.68:35181
W20240502 01:58:13.239528 8692 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:13.239737 8692 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:13.239800 8692 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:13.242807 8692 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:13.242879 8692 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:13.242969 8692 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.69
I20240502 01:58:13.247525 8692 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:35093
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.69:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.7.221.69
--webserver_port=0
--tserver_master_addrs=127.7.221.126:39109
--never_fsync=true
--heap_profile_path=/tmp/kudu.8692
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.69
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:13.248476 8692 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:13.249773 8692 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:13.258459 8699 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:13.259081 8701 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:13.259369 8698 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:13.260016 8692 server_base.cc:1034] running on GCE node
I20240502 01:58:13.260367 8692 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:13.260882 8692 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:13.262051 8692 hybrid_clock.cc:648] HybridClock initialized: now 1714615093262047 us; error 34 us; skew 500 ppm
I20240502 01:58:13.264153 8692 webserver.cc:460] Webserver started at http://127.7.221.69:40113/ using document root <none> and password file <none>
I20240502 01:58:13.264729 8692 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:13.264793 8692 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:13.264983 8692 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:13.266635 8692 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "32416365804d40db9d644b5d32301d54"
format_stamp: "Formatted at 2024-05-02 01:58:13 on dist-test-slave-dmxt"
I20240502 01:58:13.267216 8692 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "32416365804d40db9d644b5d32301d54"
format_stamp: "Formatted at 2024-05-02 01:58:13 on dist-test-slave-dmxt"
I20240502 01:58:13.270797 8692 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.001s
I20240502 01:58:13.273093 8707 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:13.274125 8692 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:13.274240 8692 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "32416365804d40db9d644b5d32301d54"
format_stamp: "Formatted at 2024-05-02 01:58:13 on dist-test-slave-dmxt"
I20240502 01:58:13.274333 8692 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:13.288866 8692 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:13.289510 8692 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:13.289672 8692 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:13.290199 8692 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:13.291180 8692 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:13.291237 8692 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:13.291296 8692 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:13.291328 8692 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:13.299997 8692 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.69:40775
I20240502 01:58:13.300046 8824 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.69:40775 every 8 connection(s)
I20240502 01:58:13.300945 8692 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 01:58:13.309352 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 8692
I20240502 01:58:13.309496 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 01:58:13.310262 8825 heartbeater.cc:347] Connected to a master server at 127.7.221.126:39109
I20240502 01:58:13.310531 8825 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:13.311228 8825 heartbeater.cc:511] Master 127.7.221.126:39109 requested a full tablet report, sending...
I20240502 01:58:13.311868 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.70:0
--local_ip_for_outbound_sockets=127.7.221.70
--webserver_interface=127.7.221.70
--webserver_port=0
--tserver_master_addrs=127.7.221.126:39109
--builtin_ntp_servers=127.7.221.84:35093
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:13.344780 8096 ts_manager.cc:194] Registered new tserver with Master: 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775)
I20240502 01:58:13.345737 8096 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.69:40965
W20240502 01:58:13.420542 8828 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:13.420778 8828 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:13.420821 8828 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:13.424000 8828 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:13.424074 8828 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:13.424139 8828 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.70
I20240502 01:58:13.427788 8828 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:35093
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.70:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.7.221.70
--webserver_port=0
--tserver_master_addrs=127.7.221.126:39109
--never_fsync=true
--heap_profile_path=/tmp/kudu.8828
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.70
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:13.428715 8828 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:13.429951 8828 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:13.438212 8834 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:13.438212 8835 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:13.439105 8837 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:13.439980 8828 server_base.cc:1034] running on GCE node
I20240502 01:58:13.440376 8828 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:13.440948 8828 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:13.442098 8828 hybrid_clock.cc:648] HybridClock initialized: now 1714615093442073 us; error 38 us; skew 500 ppm
I20240502 01:58:13.444052 8828 webserver.cc:460] Webserver started at http://127.7.221.70:39617/ using document root <none> and password file <none>
I20240502 01:58:13.444623 8828 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:13.444679 8828 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:13.444872 8828 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:13.446560 8828 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/instance:
uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
format_stamp: "Formatted at 2024-05-02 01:58:13 on dist-test-slave-dmxt"
I20240502 01:58:13.447080 8828 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal/instance:
uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
format_stamp: "Formatted at 2024-05-02 01:58:13 on dist-test-slave-dmxt"
I20240502 01:58:13.450364 8828 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 01:58:13.452490 8843 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:13.453497 8828 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:13.453598 8828 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
format_stamp: "Formatted at 2024-05-02 01:58:13 on dist-test-slave-dmxt"
I20240502 01:58:13.453681 8828 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:13.469564 8828 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:13.470182 8828 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:13.470342 8828 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:13.470902 8828 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:13.471761 8828 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:13.471812 8828 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:13.471870 8828 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:13.471895 8828 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:13.480731 8828 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.70:35887
I20240502 01:58:13.480751 8958 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.70:35887 every 8 connection(s)
I20240502 01:58:13.481688 8828 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 01:58:13.488281 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 8828
I20240502 01:58:13.488394 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240502 01:58:13.490428 8959 heartbeater.cc:347] Connected to a master server at 127.7.221.126:39109
I20240502 01:58:13.490671 8959 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:13.491273 8959 heartbeater.cc:511] Master 127.7.221.126:39109 requested a full tablet report, sending...
I20240502 01:58:13.525700 8096 ts_manager.cc:194] Registered new tserver with Master: 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:13.526564 8096 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.70:42829
I20240502 01:58:13.527458 8053 external_mini_cluster.cc:930] 6 TS(s) registered with all masters
I20240502 01:58:13.538396 8053 external_mini_cluster.cc:930] 6 TS(s) registered with all masters
I20240502 01:58:13.551236 8286 heartbeater.cc:503] Master 127.7.221.126:39109 was elected leader, sending a full tablet report...
I20240502 01:58:13.764170 8419 heartbeater.cc:503] Master 127.7.221.126:39109 was elected leader, sending a full tablet report...
I20240502 01:58:13.967546 8554 heartbeater.cc:503] Master 127.7.221.126:39109 was elected leader, sending a full tablet report...
I20240502 01:58:14.165788 8689 heartbeater.cc:503] Master 127.7.221.126:39109 was elected leader, sending a full tablet report...
I20240502 01:58:14.348022 8825 heartbeater.cc:503] Master 127.7.221.126:39109 was elected leader, sending a full tablet report...
I20240502 01:58:14.529244 8959 heartbeater.cc:503] Master 127.7.221.126:39109 was elected leader, sending a full tablet report...
I20240502 01:58:17.290875 8096 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34536:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:17.311441 8220 tablet_service.cc:1467] Processing CreateTablet for tablet 9885bd347924484186d87ba4e4c4ec2e (DEFAULT_TABLE table=rebalance_test_table_0 [id=d2f75882fa5249f09753105511998c94]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.312521 8220 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9885bd347924484186d87ba4e4c4ec2e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.312762 8892 tablet_service.cc:1467] Processing CreateTablet for tablet 9885bd347924484186d87ba4e4c4ec2e (DEFAULT_TABLE table=rebalance_test_table_0 [id=d2f75882fa5249f09753105511998c94]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.313697 8892 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9885bd347924484186d87ba4e4c4ec2e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.315688 8485 tablet_service.cc:1467] Processing CreateTablet for tablet 61ed6dd4469c477c8f6352f9c8b25576 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d2f75882fa5249f09753105511998c94]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.315690 8621 tablet_service.cc:1467] Processing CreateTablet for tablet 54f7129ce08748659fe3c3178d427d67 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d2f75882fa5249f09753105511998c94]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.316538 8621 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 54f7129ce08748659fe3c3178d427d67. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.316538 8485 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 61ed6dd4469c477c8f6352f9c8b25576. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.317677 8352 tablet_service.cc:1467] Processing CreateTablet for tablet 54f7129ce08748659fe3c3178d427d67 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d2f75882fa5249f09753105511998c94]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.318511 8352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 54f7129ce08748659fe3c3178d427d67. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.319156 8219 tablet_service.cc:1467] Processing CreateTablet for tablet 61ed6dd4469c477c8f6352f9c8b25576 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d2f75882fa5249f09753105511998c94]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.319422 8219 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 61ed6dd4469c477c8f6352f9c8b25576. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.321909 8486 tablet_service.cc:1467] Processing CreateTablet for tablet 9885bd347924484186d87ba4e4c4ec2e (DEFAULT_TABLE table=rebalance_test_table_0 [id=d2f75882fa5249f09753105511998c94]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.322183 8486 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9885bd347924484186d87ba4e4c4ec2e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.325632 8987 tablet_bootstrap.cc:492] T 54f7129ce08748659fe3c3178d427d67 P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:17.326428 8984 tablet_bootstrap.cc:492] T 9885bd347924484186d87ba4e4c4ec2e P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:17.326495 8985 tablet_bootstrap.cc:492] T 61ed6dd4469c477c8f6352f9c8b25576 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:17.328467 8985 tablet_bootstrap.cc:654] T 61ed6dd4469c477c8f6352f9c8b25576 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.328474 8984 tablet_bootstrap.cc:654] T 9885bd347924484186d87ba4e4c4ec2e P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.329270 8985 log.cc:824] T 61ed6dd4469c477c8f6352f9c8b25576 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:17.329267 8984 log.cc:824] T 9885bd347924484186d87ba4e4c4ec2e P 1caeef741efa452fa9bb1fd3d47e65eb: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:17.329447 8987 tablet_bootstrap.cc:654] T 54f7129ce08748659fe3c3178d427d67 P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.330185 8987 log.cc:824] T 54f7129ce08748659fe3c3178d427d67 P ef9a99ad58b84fd2ada7f035f27361b1: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:17.329989 8756 tablet_service.cc:1467] Processing CreateTablet for tablet 61ed6dd4469c477c8f6352f9c8b25576 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d2f75882fa5249f09753105511998c94]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.329854 8757 tablet_service.cc:1467] Processing CreateTablet for tablet 54f7129ce08748659fe3c3178d427d67 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d2f75882fa5249f09753105511998c94]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.330850 8988 tablet_bootstrap.cc:492] T 9885bd347924484186d87ba4e4c4ec2e P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:17.330948 8757 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 54f7129ce08748659fe3c3178d427d67. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.331156 8986 tablet_bootstrap.cc:492] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:17.331991 8987 tablet_bootstrap.cc:492] T 54f7129ce08748659fe3c3178d427d67 P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:17.332171 8987 ts_tablet_manager.cc:1367] T 54f7129ce08748659fe3c3178d427d67 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.007s user 0.003s sys 0.000s
I20240502 01:58:17.332789 8988 tablet_bootstrap.cc:654] T 9885bd347924484186d87ba4e4c4ec2e P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.333129 8986 tablet_bootstrap.cc:654] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.333570 8988 log.cc:824] T 9885bd347924484186d87ba4e4c4ec2e P e325acceaed541869d43c1f762530c2e: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:17.333837 8986 log.cc:824] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:17.333916 8756 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 61ed6dd4469c477c8f6352f9c8b25576. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.335086 8988 tablet_bootstrap.cc:492] T 9885bd347924484186d87ba4e4c4ec2e P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:17.335242 8988 ts_tablet_manager.cc:1367] T 9885bd347924484186d87ba4e4c4ec2e P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240502 01:58:17.335376 8984 tablet_bootstrap.cc:492] T 9885bd347924484186d87ba4e4c4ec2e P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:17.335513 8984 ts_tablet_manager.cc:1367] T 9885bd347924484186d87ba4e4c4ec2e P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.009s user 0.004s sys 0.000s
I20240502 01:58:17.335608 8985 tablet_bootstrap.cc:492] T 61ed6dd4469c477c8f6352f9c8b25576 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:17.335749 8985 ts_tablet_manager.cc:1367] T 61ed6dd4469c477c8f6352f9c8b25576 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.009s user 0.004s sys 0.000s
I20240502 01:58:17.338724 8988 raft_consensus.cc:348] T 9885bd347924484186d87ba4e4c4ec2e P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.338788 8984 raft_consensus.cc:348] T 9885bd347924484186d87ba4e4c4ec2e P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.338917 8988 raft_consensus.cc:374] T 9885bd347924484186d87ba4e4c4ec2e P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.338923 8984 raft_consensus.cc:374] T 9885bd347924484186d87ba4e4c4ec2e P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.338963 8988 raft_consensus.cc:729] T 9885bd347924484186d87ba4e4c4ec2e P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.338970 8984 raft_consensus.cc:729] T 9885bd347924484186d87ba4e4c4ec2e P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.339143 8985 raft_consensus.cc:348] T 61ed6dd4469c477c8f6352f9c8b25576 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:17.339326 8985 raft_consensus.cc:374] T 61ed6dd4469c477c8f6352f9c8b25576 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.339392 8985 raft_consensus.cc:729] T 61ed6dd4469c477c8f6352f9c8b25576 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.339409 8984 consensus_queue.cc:260] T 9885bd347924484186d87ba4e4c4ec2e P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.339409 8988 consensus_queue.cc:260] T 9885bd347924484186d87ba4e4c4ec2e P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.339753 8985 consensus_queue.cc:260] T 61ed6dd4469c477c8f6352f9c8b25576 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:17.340189 8984 ts_tablet_manager.cc:1398] T 9885bd347924484186d87ba4e4c4ec2e P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s
I20240502 01:58:17.340198 8988 ts_tablet_manager.cc:1398] T 9885bd347924484186d87ba4e4c4ec2e P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s
I20240502 01:58:17.340502 8988 tablet_bootstrap.cc:492] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:17.340535 8985 ts_tablet_manager.cc:1398] T 61ed6dd4469c477c8f6352f9c8b25576 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s
I20240502 01:58:17.340847 8985 tablet_bootstrap.cc:492] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:17.341733 8985 tablet_bootstrap.cc:654] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.342192 8988 tablet_bootstrap.cc:654] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.343971 8987 raft_consensus.cc:348] T 54f7129ce08748659fe3c3178d427d67 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:17.344153 8987 raft_consensus.cc:374] T 54f7129ce08748659fe3c3178d427d67 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.344204 8987 raft_consensus.cc:729] T 54f7129ce08748659fe3c3178d427d67 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.344656 8994 tablet_bootstrap.cc:492] T 61ed6dd4469c477c8f6352f9c8b25576 P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:17.344687 8987 consensus_queue.cc:260] T 54f7129ce08748659fe3c3178d427d67 P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:17.344772 8986 tablet_bootstrap.cc:492] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:17.344939 8986 ts_tablet_manager.cc:1367] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.014s user 0.003s sys 0.000s
I20240502 01:58:17.345511 8987 ts_tablet_manager.cc:1398] T 54f7129ce08748659fe3c3178d427d67 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.013s user 0.006s sys 0.000s
I20240502 01:58:17.346442 8994 tablet_bootstrap.cc:654] T 61ed6dd4469c477c8f6352f9c8b25576 P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.347002 8994 log.cc:824] T 61ed6dd4469c477c8f6352f9c8b25576 P 32416365804d40db9d644b5d32301d54: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:17.348202 8985 tablet_bootstrap.cc:492] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:17.348309 8985 ts_tablet_manager.cc:1367] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 01:58:17.348455 8994 tablet_bootstrap.cc:492] T 61ed6dd4469c477c8f6352f9c8b25576 P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:17.348618 8994 ts_tablet_manager.cc:1367] T 61ed6dd4469c477c8f6352f9c8b25576 P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 01:58:17.348554 8986 raft_consensus.cc:348] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:17.348740 8986 raft_consensus.cc:374] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.348827 8986 raft_consensus.cc:729] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.348886 8985 raft_consensus.cc:348] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.348978 8985 raft_consensus.cc:374] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.349009 8985 raft_consensus.cc:729] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.349129 8985 consensus_queue.cc:260] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.349370 8985 ts_tablet_manager.cc:1398] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:17.349608 8986 consensus_queue.cc:260] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:17.350307 8988 tablet_bootstrap.cc:492] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:17.350425 8988 ts_tablet_manager.cc:1367] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.000s
I20240502 01:58:17.350647 8986 ts_tablet_manager.cc:1398] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.006s user 0.006s sys 0.000s
I20240502 01:58:17.351056 8988 raft_consensus.cc:348] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:17.351163 8988 raft_consensus.cc:374] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.351194 8988 raft_consensus.cc:729] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.351312 8988 consensus_queue.cc:260] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:17.351552 8988 ts_tablet_manager.cc:1398] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:17.353296 8994 raft_consensus.cc:348] T 61ed6dd4469c477c8f6352f9c8b25576 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:17.353488 8994 raft_consensus.cc:374] T 61ed6dd4469c477c8f6352f9c8b25576 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.353539 8994 raft_consensus.cc:729] T 61ed6dd4469c477c8f6352f9c8b25576 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.353893 8994 consensus_queue.cc:260] T 61ed6dd4469c477c8f6352f9c8b25576 P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:17.354696 8994 ts_tablet_manager.cc:1398] T 61ed6dd4469c477c8f6352f9c8b25576 P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.006s user 0.006s sys 0.000s
I20240502 01:58:17.354971 8994 tablet_bootstrap.cc:492] T 54f7129ce08748659fe3c3178d427d67 P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:17.355830 8994 tablet_bootstrap.cc:654] T 54f7129ce08748659fe3c3178d427d67 P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.356871 8994 tablet_bootstrap.cc:492] T 54f7129ce08748659fe3c3178d427d67 P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:17.356945 8994 ts_tablet_manager.cc:1367] T 54f7129ce08748659fe3c3178d427d67 P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:17.357378 8994 raft_consensus.cc:348] T 54f7129ce08748659fe3c3178d427d67 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:17.357465 8994 raft_consensus.cc:374] T 54f7129ce08748659fe3c3178d427d67 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.357496 8994 raft_consensus.cc:729] T 54f7129ce08748659fe3c3178d427d67 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.357609 8994 consensus_queue.cc:260] T 54f7129ce08748659fe3c3178d427d67 P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:17.357820 8994 ts_tablet_manager.cc:1398] T 54f7129ce08748659fe3c3178d427d67 P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
W20240502 01:58:17.358487 8690 tablet.cc:2369] T 54f7129ce08748659fe3c3178d427d67 P ef9a99ad58b84fd2ada7f035f27361b1: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 01:58:17.388836 8996 raft_consensus.cc:482] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:17.389024 8996 raft_consensus.cc:504] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:17.390076 8996 leader_election.cc:288] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775)
I20240502 01:58:17.393833 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "61ed6dd4469c477c8f6352f9c8b25576" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" is_pre_election: true
I20240502 01:58:17.393833 8777 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "61ed6dd4469c477c8f6352f9c8b25576" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54" is_pre_election: true
I20240502 01:58:17.394095 8240 raft_consensus.cc:2454] T 61ed6dd4469c477c8f6352f9c8b25576 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 0.
I20240502 01:58:17.394098 8777 raft_consensus.cc:2454] T 61ed6dd4469c477c8f6352f9c8b25576 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 0.
I20240502 01:58:17.394512 8441 leader_election.cc:302] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e325acceaed541869d43c1f762530c2e, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:17.394759 8996 raft_consensus.cc:2781] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:17.394829 8996 raft_consensus.cc:482] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:17.394856 8996 raft_consensus.cc:3037] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.395764 8996 raft_consensus.cc:504] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:17.396109 8996 leader_election.cc:288] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 election: Requested vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775)
I20240502 01:58:17.396467 8777 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "61ed6dd4469c477c8f6352f9c8b25576" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54"
I20240502 01:58:17.396463 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "61ed6dd4469c477c8f6352f9c8b25576" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
I20240502 01:58:17.396564 8240 raft_consensus.cc:3037] T 61ed6dd4469c477c8f6352f9c8b25576 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.396574 8777 raft_consensus.cc:3037] T 61ed6dd4469c477c8f6352f9c8b25576 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.397416 8240 raft_consensus.cc:2454] T 61ed6dd4469c477c8f6352f9c8b25576 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 1.
I20240502 01:58:17.397504 8777 raft_consensus.cc:2454] T 61ed6dd4469c477c8f6352f9c8b25576 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 1.
I20240502 01:58:17.397676 8441 leader_election.cc:302] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e325acceaed541869d43c1f762530c2e, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:17.397826 8996 raft_consensus.cc:2781] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:17.398054 8996 raft_consensus.cc:686] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [term 1 LEADER]: Becoming Leader. State: Replica: e325acceaed541869d43c1f762530c2e, State: Running, Role: LEADER
I20240502 01:58:17.398360 8996 consensus_queue.cc:237] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:17.401351 8096 catalog_manager.cc:5484] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e reported cstate change: term changed from 0 to 1, leader changed from <none> to e325acceaed541869d43c1f762530c2e (127.7.221.67). New cstate: current_term: 1 leader_uuid: "e325acceaed541869d43c1f762530c2e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:17.425902 8555 tablet.cc:2369] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 01:58:17.438086 8997 raft_consensus.cc:482] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:17.438216 8997 raft_consensus.cc:504] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.439239 8997 leader_election.cc:288] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e325acceaed541869d43c1f762530c2e (127.7.221.67:42267), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:17.444110 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9885bd347924484186d87ba4e4c4ec2e" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" is_pre_election: true
I20240502 01:58:17.444307 8507 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9885bd347924484186d87ba4e4c4ec2e" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e" is_pre_election: true
I20240502 01:58:17.444427 8912 raft_consensus.cc:2454] T 9885bd347924484186d87ba4e4c4ec2e P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 0.
I20240502 01:58:17.444528 8507 raft_consensus.cc:2454] T 9885bd347924484186d87ba4e4c4ec2e P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 0.
I20240502 01:58:17.445247 8172 leader_election.cc:302] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, e325acceaed541869d43c1f762530c2e, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:17.445497 8997 raft_consensus.cc:2781] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:17.445569 8997 raft_consensus.cc:482] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:17.445595 8997 raft_consensus.cc:3037] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.446445 8997 raft_consensus.cc:504] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.446806 8997 leader_election.cc:288] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 election: Requested vote from peers e325acceaed541869d43c1f762530c2e (127.7.221.67:42267), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:17.447120 8507 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9885bd347924484186d87ba4e4c4ec2e" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e"
I20240502 01:58:17.447125 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9885bd347924484186d87ba4e4c4ec2e" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
I20240502 01:58:17.447211 8912 raft_consensus.cc:3037] T 9885bd347924484186d87ba4e4c4ec2e P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.447224 8507 raft_consensus.cc:3037] T 9885bd347924484186d87ba4e4c4ec2e P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.447942 8507 raft_consensus.cc:2454] T 9885bd347924484186d87ba4e4c4ec2e P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 1.
I20240502 01:58:17.448009 8912 raft_consensus.cc:2454] T 9885bd347924484186d87ba4e4c4ec2e P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 1.
I20240502 01:58:17.448243 8173 leader_election.cc:302] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e325acceaed541869d43c1f762530c2e, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:17.448412 8997 raft_consensus.cc:2781] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:17.448602 8997 raft_consensus.cc:686] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 LEADER]: Becoming Leader. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Running, Role: LEADER
I20240502 01:58:17.448897 8997 consensus_queue.cc:237] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.451676 8096 catalog_manager.cc:5484] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 reported cstate change: term changed from 0 to 1, leader changed from <none> to e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:17.457607 8420 tablet.cc:2369] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 01:58:17.485743 8960 tablet.cc:2369] T 9885bd347924484186d87ba4e4c4ec2e P 1caeef741efa452fa9bb1fd3d47e65eb: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 01:58:17.502835 8287 tablet.cc:2369] T 61ed6dd4469c477c8f6352f9c8b25576 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 01:58:17.511031 9002 raft_consensus.cc:482] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:17.511248 9002 raft_consensus.cc:504] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:17.512323 9002 leader_election.cc:288] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775), ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021)
I20240502 01:58:17.516489 8777 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "54f7129ce08748659fe3c3178d427d67" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54" is_pre_election: true
I20240502 01:58:17.516623 8777 raft_consensus.cc:2454] T 54f7129ce08748659fe3c3178d427d67 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 73ee70f488334b8f8ffb83aef6456e15 in term 0.
I20240502 01:58:17.516731 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "54f7129ce08748659fe3c3178d427d67" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" is_pre_election: true
I20240502 01:58:17.516974 8305 leader_election.cc:302] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, 73ee70f488334b8f8ffb83aef6456e15; no voters:
I20240502 01:58:17.517024 8642 raft_consensus.cc:2454] T 54f7129ce08748659fe3c3178d427d67 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 73ee70f488334b8f8ffb83aef6456e15 in term 0.
I20240502 01:58:17.517225 9002 raft_consensus.cc:2781] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:17.517294 9002 raft_consensus.cc:482] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:17.517349 9002 raft_consensus.cc:3037] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.518183 9002 raft_consensus.cc:504] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:17.518483 9002 leader_election.cc:288] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 election: Requested vote from peers 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775), ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021)
I20240502 01:58:17.518838 8777 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "54f7129ce08748659fe3c3178d427d67" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54"
I20240502 01:58:17.518910 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "54f7129ce08748659fe3c3178d427d67" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1"
I20240502 01:58:17.518960 8777 raft_consensus.cc:3037] T 54f7129ce08748659fe3c3178d427d67 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.519030 8642 raft_consensus.cc:3037] T 54f7129ce08748659fe3c3178d427d67 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.519753 8777 raft_consensus.cc:2454] T 54f7129ce08748659fe3c3178d427d67 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 73ee70f488334b8f8ffb83aef6456e15 in term 1.
I20240502 01:58:17.519850 8642 raft_consensus.cc:2454] T 54f7129ce08748659fe3c3178d427d67 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 73ee70f488334b8f8ffb83aef6456e15 in term 1.
I20240502 01:58:17.520031 8305 leader_election.cc:302] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, 73ee70f488334b8f8ffb83aef6456e15; no voters:
I20240502 01:58:17.520284 9002 raft_consensus.cc:2781] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:17.520480 9002 raft_consensus.cc:686] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [term 1 LEADER]: Becoming Leader. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Running, Role: LEADER
I20240502 01:58:17.520792 9002 consensus_queue.cc:237] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:17.523885 8096 catalog_manager.cc:5484] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 reported cstate change: term changed from 0 to 1, leader changed from <none> to 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "73ee70f488334b8f8ffb83aef6456e15" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: HEALTHY } } }
W20240502 01:58:17.555088 8826 tablet.cc:2369] T 54f7129ce08748659fe3c3178d427d67 P 32416365804d40db9d644b5d32301d54: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 01:58:17.646449 9019 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:17.646749 9019 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:17.649793 9019 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_0'
I20240502 01:58:17.739198 8777 raft_consensus.cc:1261] T 54f7129ce08748659fe3c3178d427d67 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Refusing update from remote peer 73ee70f488334b8f8ffb83aef6456e15: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:17.739295 8642 raft_consensus.cc:1261] T 54f7129ce08748659fe3c3178d427d67 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Refusing update from remote peer 73ee70f488334b8f8ffb83aef6456e15: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:17.739912 8912 raft_consensus.cc:1261] T 9885bd347924484186d87ba4e4c4ec2e P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Refusing update from remote peer e7a8991bb9ff47489bf8ccbaf5ca7fa9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:17.740156 9016 consensus_queue.cc:1035] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:17.740593 9016 consensus_queue.cc:1035] T 54f7129ce08748659fe3c3178d427d67 P 73ee70f488334b8f8ffb83aef6456e15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:17.740656 9012 consensus_queue.cc:1035] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:17.741654 8507 raft_consensus.cc:1261] T 9885bd347924484186d87ba4e4c4ec2e P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Refusing update from remote peer e7a8991bb9ff47489bf8ccbaf5ca7fa9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:17.742168 8997 consensus_queue.cc:1035] T 9885bd347924484186d87ba4e4c4ec2e P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:17.749186 8776 raft_consensus.cc:1261] T 61ed6dd4469c477c8f6352f9c8b25576 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Refusing update from remote peer e325acceaed541869d43c1f762530c2e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:17.749366 8240 raft_consensus.cc:1261] T 61ed6dd4469c477c8f6352f9c8b25576 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Refusing update from remote peer e325acceaed541869d43c1f762530c2e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:17.749902 9008 consensus_queue.cc:1035] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [LEADER]: Connected to new peer: Peer: permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:17.750281 9008 consensus_queue.cc:1035] T 61ed6dd4469c477c8f6352f9c8b25576 P e325acceaed541869d43c1f762530c2e [LEADER]: Connected to new peer: Peer: permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:17.757187 9054 mvcc.cc:205] Tried to move back new op lower bound from 7023063440333647872 to 7023063439152939008. Current Snapshot: MvccSnapshot[applied={T|T < 7023063440333647872}]
I20240502 01:58:17.760252 9060 mvcc.cc:205] Tried to move back new op lower bound from 7023063440333647872 to 7023063439152939008. Current Snapshot: MvccSnapshot[applied={T|T < 7023063440333647872}]
I20240502 01:58:17.762794 9052 mvcc.cc:205] Tried to move back new op lower bound from 7023063440376033280 to 7023063438946390016. Current Snapshot: MvccSnapshot[applied={T|T < 7023063440376033280}]
I20240502 01:58:17.783581 9053 mvcc.cc:205] Tried to move back new op lower bound from 7023063440332177408 to 7023063439447519232. Current Snapshot: MvccSnapshot[applied={T|T < 7023063440332177408}]
I20240502 01:58:17.785365 9055 mvcc.cc:205] Tried to move back new op lower bound from 7023063440332177408 to 7023063439447519232. Current Snapshot: MvccSnapshot[applied={T|T < 7023063440332177408}]
INSERT report
rows total: 2000
time total: 118.416 ms
time per row: 0.0592079 ms
I20240502 01:58:17.814978 8093 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34536:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:17.820444 8621 tablet_service.cc:1467] Processing CreateTablet for tablet c49a55dc4dbf48da905d7cfa0bf2fb2a (DEFAULT_TABLE table=rebalance_test_table_1 [id=2d7e4ef7009845eda863712530cd019b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.820541 8352 tablet_service.cc:1467] Processing CreateTablet for tablet c49a55dc4dbf48da905d7cfa0bf2fb2a (DEFAULT_TABLE table=rebalance_test_table_1 [id=2d7e4ef7009845eda863712530cd019b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.820797 8621 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c49a55dc4dbf48da905d7cfa0bf2fb2a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.820858 8352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c49a55dc4dbf48da905d7cfa0bf2fb2a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.821030 8892 tablet_service.cc:1467] Processing CreateTablet for tablet c49a55dc4dbf48da905d7cfa0bf2fb2a (DEFAULT_TABLE table=rebalance_test_table_1 [id=2d7e4ef7009845eda863712530cd019b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.821296 8892 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c49a55dc4dbf48da905d7cfa0bf2fb2a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.822671 8987 tablet_bootstrap.cc:492] T c49a55dc4dbf48da905d7cfa0bf2fb2a P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:17.823074 8986 tablet_bootstrap.cc:492] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:17.823514 8987 tablet_bootstrap.cc:654] T c49a55dc4dbf48da905d7cfa0bf2fb2a P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.823582 8891 tablet_service.cc:1467] Processing CreateTablet for tablet ed11274a6a2e4c07b272fe104533ea62 (DEFAULT_TABLE table=rebalance_test_table_1 [id=2d7e4ef7009845eda863712530cd019b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.823808 8891 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ed11274a6a2e4c07b272fe104533ea62. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.823899 8986 tablet_bootstrap.cc:654] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.824728 8485 tablet_service.cc:1467] Processing CreateTablet for tablet 851d391e5df146f9858f5d07bf48a81b (DEFAULT_TABLE table=rebalance_test_table_1 [id=2d7e4ef7009845eda863712530cd019b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.824950 8485 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 851d391e5df146f9858f5d07bf48a81b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.825068 8986 tablet_bootstrap.cc:492] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:17.825143 8986 ts_tablet_manager.cc:1367] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:17.825742 8984 tablet_bootstrap.cc:492] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:17.825752 8986 raft_consensus.cc:348] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.825860 8986 raft_consensus.cc:374] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.825892 8986 raft_consensus.cc:729] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.826030 8986 consensus_queue.cc:260] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.826684 8988 tablet_bootstrap.cc:492] T 851d391e5df146f9858f5d07bf48a81b P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:17.826771 8984 tablet_bootstrap.cc:654] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.827512 8986 ts_tablet_manager.cc:1398] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:17.827525 8988 tablet_bootstrap.cc:654] T 851d391e5df146f9858f5d07bf48a81b P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.828009 8621 tablet_service.cc:1467] Processing CreateTablet for tablet ed11274a6a2e4c07b272fe104533ea62 (DEFAULT_TABLE table=rebalance_test_table_1 [id=2d7e4ef7009845eda863712530cd019b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.828279 8621 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ed11274a6a2e4c07b272fe104533ea62. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.828430 8987 tablet_bootstrap.cc:492] T c49a55dc4dbf48da905d7cfa0bf2fb2a P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:17.828524 8987 ts_tablet_manager.cc:1367] T c49a55dc4dbf48da905d7cfa0bf2fb2a P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 01:58:17.828596 8988 tablet_bootstrap.cc:492] T 851d391e5df146f9858f5d07bf48a81b P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:17.828680 8988 ts_tablet_manager.cc:1367] T 851d391e5df146f9858f5d07bf48a81b P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 01:58:17.829069 8987 raft_consensus.cc:348] T c49a55dc4dbf48da905d7cfa0bf2fb2a P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.829054 8988 raft_consensus.cc:348] T 851d391e5df146f9858f5d07bf48a81b P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:17.829164 8988 raft_consensus.cc:374] T 851d391e5df146f9858f5d07bf48a81b P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.829164 8987 raft_consensus.cc:374] T c49a55dc4dbf48da905d7cfa0bf2fb2a P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.829193 8988 raft_consensus.cc:729] T 851d391e5df146f9858f5d07bf48a81b P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.829193 8987 raft_consensus.cc:729] T c49a55dc4dbf48da905d7cfa0bf2fb2a P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.829322 8988 consensus_queue.cc:260] T 851d391e5df146f9858f5d07bf48a81b P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:17.829322 8987 consensus_queue.cc:260] T c49a55dc4dbf48da905d7cfa0bf2fb2a P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.829572 8988 ts_tablet_manager.cc:1398] T 851d391e5df146f9858f5d07bf48a81b P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:17.829593 8987 ts_tablet_manager.cc:1398] T c49a55dc4dbf48da905d7cfa0bf2fb2a P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:17.829944 8987 tablet_bootstrap.cc:492] T ed11274a6a2e4c07b272fe104533ea62 P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:17.830847 8987 tablet_bootstrap.cc:654] T ed11274a6a2e4c07b272fe104533ea62 P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.830940 8757 tablet_service.cc:1467] Processing CreateTablet for tablet 851d391e5df146f9858f5d07bf48a81b (DEFAULT_TABLE table=rebalance_test_table_1 [id=2d7e4ef7009845eda863712530cd019b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.831221 8757 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 851d391e5df146f9858f5d07bf48a81b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.832788 8994 tablet_bootstrap.cc:492] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:17.833484 8220 tablet_service.cc:1467] Processing CreateTablet for tablet 851d391e5df146f9858f5d07bf48a81b (DEFAULT_TABLE table=rebalance_test_table_1 [id=2d7e4ef7009845eda863712530cd019b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.833598 8994 tablet_bootstrap.cc:654] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.833611 8984 tablet_bootstrap.cc:492] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:17.833709 8984 ts_tablet_manager.cc:1367] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.000s
I20240502 01:58:17.833832 8220 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 851d391e5df146f9858f5d07bf48a81b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.834391 8984 raft_consensus.cc:348] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.834497 8984 raft_consensus.cc:374] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.834535 8984 raft_consensus.cc:729] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.834656 8984 consensus_queue.cc:260] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.834901 8984 ts_tablet_manager.cc:1398] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:17.835060 8984 tablet_bootstrap.cc:492] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:17.835724 8984 tablet_bootstrap.cc:654] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.835826 8985 tablet_bootstrap.cc:492] T 851d391e5df146f9858f5d07bf48a81b P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:17.836473 8352 tablet_service.cc:1467] Processing CreateTablet for tablet ed11274a6a2e4c07b272fe104533ea62 (DEFAULT_TABLE table=rebalance_test_table_1 [id=2d7e4ef7009845eda863712530cd019b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:17.836681 8984 tablet_bootstrap.cc:492] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:17.836707 8985 tablet_bootstrap.cc:654] T 851d391e5df146f9858f5d07bf48a81b P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.836764 8984 ts_tablet_manager.cc:1367] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 01:58:17.836846 8352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ed11274a6a2e4c07b272fe104533ea62. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:17.837160 8984 raft_consensus.cc:348] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.837255 8984 raft_consensus.cc:374] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.837281 8984 raft_consensus.cc:729] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.837388 8984 consensus_queue.cc:260] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.837638 8984 ts_tablet_manager.cc:1398] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:17.838227 8994 tablet_bootstrap.cc:492] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:17.838310 8994 ts_tablet_manager.cc:1367] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 01:58:17.838799 8986 tablet_bootstrap.cc:492] T ed11274a6a2e4c07b272fe104533ea62 P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:17.838779 8994 raft_consensus.cc:348] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:17.838881 8994 raft_consensus.cc:374] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.838914 8994 raft_consensus.cc:729] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.839052 8994 consensus_queue.cc:260] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:17.839329 8994 ts_tablet_manager.cc:1398] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:17.839504 8987 tablet_bootstrap.cc:492] T ed11274a6a2e4c07b272fe104533ea62 P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:17.839589 8987 ts_tablet_manager.cc:1367] T ed11274a6a2e4c07b272fe104533ea62 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.010s user 0.001s sys 0.000s
I20240502 01:58:17.839704 8986 tablet_bootstrap.cc:654] T ed11274a6a2e4c07b272fe104533ea62 P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:17.840049 8987 raft_consensus.cc:348] T ed11274a6a2e4c07b272fe104533ea62 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.840180 8987 raft_consensus.cc:374] T ed11274a6a2e4c07b272fe104533ea62 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.840216 8987 raft_consensus.cc:729] T ed11274a6a2e4c07b272fe104533ea62 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.840340 8987 consensus_queue.cc:260] T ed11274a6a2e4c07b272fe104533ea62 P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.840587 8987 ts_tablet_manager.cc:1398] T ed11274a6a2e4c07b272fe104533ea62 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:17.841490 8986 tablet_bootstrap.cc:492] T ed11274a6a2e4c07b272fe104533ea62 P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:17.841593 8986 ts_tablet_manager.cc:1367] T ed11274a6a2e4c07b272fe104533ea62 P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:17.841590 8985 tablet_bootstrap.cc:492] T 851d391e5df146f9858f5d07bf48a81b P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:17.841672 8985 ts_tablet_manager.cc:1367] T 851d391e5df146f9858f5d07bf48a81b P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 01:58:17.842072 8985 raft_consensus.cc:348] T 851d391e5df146f9858f5d07bf48a81b P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:17.842058 8986 raft_consensus.cc:348] T ed11274a6a2e4c07b272fe104533ea62 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.842165 8985 raft_consensus.cc:374] T 851d391e5df146f9858f5d07bf48a81b P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.842168 8986 raft_consensus.cc:374] T ed11274a6a2e4c07b272fe104533ea62 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:17.842186 8985 raft_consensus.cc:729] T 851d391e5df146f9858f5d07bf48a81b P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.842204 8986 raft_consensus.cc:729] T ed11274a6a2e4c07b272fe104533ea62 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:17.842298 8985 consensus_queue.cc:260] T 851d391e5df146f9858f5d07bf48a81b P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:17.842350 8986 consensus_queue.cc:260] T ed11274a6a2e4c07b272fe104533ea62 P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.843698 8985 ts_tablet_manager.cc:1398] T 851d391e5df146f9858f5d07bf48a81b P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:17.843817 8986 ts_tablet_manager.cc:1398] T ed11274a6a2e4c07b272fe104533ea62 P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:17.863809 8995 raft_consensus.cc:482] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:17.863932 8995 raft_consensus.cc:504] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.864991 8995 leader_election.cc:288] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021)
I20240502 01:58:17.869366 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ed11274a6a2e4c07b272fe104533ea62" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" is_pre_election: true
I20240502 01:58:17.869514 8642 raft_consensus.cc:2454] T ed11274a6a2e4c07b272fe104533ea62 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 0.
I20240502 01:58:17.869885 8844 leader_election.cc:302] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:17.870167 8995 raft_consensus.cc:2781] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:17.870230 8995 raft_consensus.cc:482] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:17.870273 8995 raft_consensus.cc:3037] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.870400 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ed11274a6a2e4c07b272fe104533ea62" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15" is_pre_election: true
I20240502 01:58:17.870625 8373 raft_consensus.cc:2454] T ed11274a6a2e4c07b272fe104533ea62 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 0.
I20240502 01:58:17.871256 8995 raft_consensus.cc:504] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.871557 8995 leader_election.cc:288] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 election: Requested vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021)
I20240502 01:58:17.871855 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ed11274a6a2e4c07b272fe104533ea62" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15"
I20240502 01:58:17.871882 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ed11274a6a2e4c07b272fe104533ea62" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1"
I20240502 01:58:17.871973 8373 raft_consensus.cc:3037] T ed11274a6a2e4c07b272fe104533ea62 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.871985 8642 raft_consensus.cc:3037] T ed11274a6a2e4c07b272fe104533ea62 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.872968 8373 raft_consensus.cc:2454] T ed11274a6a2e4c07b272fe104533ea62 P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 1.
I20240502 01:58:17.872968 8642 raft_consensus.cc:2454] T ed11274a6a2e4c07b272fe104533ea62 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 1.
I20240502 01:58:17.873292 8845 leader_election.cc:302] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, 73ee70f488334b8f8ffb83aef6456e15; no voters:
I20240502 01:58:17.873462 8995 raft_consensus.cc:2781] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:17.873683 8995 raft_consensus.cc:686] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 LEADER]: Becoming Leader. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Running, Role: LEADER
I20240502 01:58:17.873951 8995 consensus_queue.cc:237] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.876259 8097 catalog_manager.cc:5484] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb reported cstate change: term changed from 0 to 1, leader changed from <none> to 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70). New cstate: current_term: 1 leader_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: HEALTHY } } }
I20240502 01:58:17.941109 9003 raft_consensus.cc:482] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:17.941267 9003 raft_consensus.cc:504] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:17.942261 9003 leader_election.cc:288] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), e325acceaed541869d43c1f762530c2e (127.7.221.67:42267)
I20240502 01:58:17.946414 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "851d391e5df146f9858f5d07bf48a81b" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" is_pre_election: true
I20240502 01:58:17.946414 8507 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "851d391e5df146f9858f5d07bf48a81b" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e" is_pre_election: true
I20240502 01:58:17.946569 8240 raft_consensus.cc:2454] T 851d391e5df146f9858f5d07bf48a81b P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 0.
I20240502 01:58:17.946569 8507 raft_consensus.cc:2454] T 851d391e5df146f9858f5d07bf48a81b P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 0.
I20240502 01:58:17.946954 8710 leader_election.cc:302] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:17.947196 9003 raft_consensus.cc:2781] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:17.947258 9003 raft_consensus.cc:482] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:17.947283 9003 raft_consensus.cc:3037] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.948114 9003 raft_consensus.cc:504] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:17.948441 9003 leader_election.cc:288] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 election: Requested vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), e325acceaed541869d43c1f762530c2e (127.7.221.67:42267)
I20240502 01:58:17.948779 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "851d391e5df146f9858f5d07bf48a81b" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
I20240502 01:58:17.948802 8507 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "851d391e5df146f9858f5d07bf48a81b" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e"
I20240502 01:58:17.948904 8507 raft_consensus.cc:3037] T 851d391e5df146f9858f5d07bf48a81b P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.948904 8240 raft_consensus.cc:3037] T 851d391e5df146f9858f5d07bf48a81b P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.949857 8240 raft_consensus.cc:2454] T 851d391e5df146f9858f5d07bf48a81b P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 1.
I20240502 01:58:17.949857 8507 raft_consensus.cc:2454] T 851d391e5df146f9858f5d07bf48a81b P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 1.
I20240502 01:58:17.950189 8710 leader_election.cc:302] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:17.950385 9003 raft_consensus.cc:2781] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:17.950618 9003 raft_consensus.cc:686] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [term 1 LEADER]: Becoming Leader. State: Replica: 32416365804d40db9d644b5d32301d54, State: Running, Role: LEADER
I20240502 01:58:17.950945 9003 consensus_queue.cc:237] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:17.953294 8097 catalog_manager.cc:5484] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 reported cstate change: term changed from 0 to 1, leader changed from <none> to 32416365804d40db9d644b5d32301d54 (127.7.221.69). New cstate: current_term: 1 leader_uuid: "32416365804d40db9d644b5d32301d54" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:17.962757 8995 raft_consensus.cc:482] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:17.962867 8995 raft_consensus.cc:504] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.963205 8995 leader_election.cc:288] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021), 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503)
I20240502 01:58:17.963641 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c49a55dc4dbf48da905d7cfa0bf2fb2a" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" is_pre_election: true
I20240502 01:58:17.963802 8642 raft_consensus.cc:2454] T c49a55dc4dbf48da905d7cfa0bf2fb2a P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 0.
I20240502 01:58:17.963838 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c49a55dc4dbf48da905d7cfa0bf2fb2a" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15" is_pre_election: true
I20240502 01:58:17.964001 8373 raft_consensus.cc:2454] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 0.
I20240502 01:58:17.964134 8844 leader_election.cc:302] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:17.964305 8995 raft_consensus.cc:2781] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:17.964391 8995 raft_consensus.cc:482] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:17.964428 8995 raft_consensus.cc:3037] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.965180 8995 raft_consensus.cc:504] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.965461 8995 leader_election.cc:288] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 election: Requested vote from peers ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021), 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503)
I20240502 01:58:17.965859 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c49a55dc4dbf48da905d7cfa0bf2fb2a" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1"
I20240502 01:58:17.965865 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c49a55dc4dbf48da905d7cfa0bf2fb2a" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15"
I20240502 01:58:17.965957 8373 raft_consensus.cc:3037] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.965973 8642 raft_consensus.cc:3037] T c49a55dc4dbf48da905d7cfa0bf2fb2a P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:17.966713 8373 raft_consensus.cc:2454] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 1.
I20240502 01:58:17.966738 8642 raft_consensus.cc:2454] T c49a55dc4dbf48da905d7cfa0bf2fb2a P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 1.
I20240502 01:58:17.966980 8845 leader_election.cc:302] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, 73ee70f488334b8f8ffb83aef6456e15; no voters:
I20240502 01:58:17.967159 8995 raft_consensus.cc:2781] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:17.967248 8995 raft_consensus.cc:686] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 LEADER]: Becoming Leader. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Running, Role: LEADER
I20240502 01:58:17.967417 8995 consensus_queue.cc:237] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:17.969024 8097 catalog_manager.cc:5484] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb reported cstate change: term changed from 0 to 1, leader changed from <none> to 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70). New cstate: current_term: 1 leader_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: HEALTHY } } }
W20240502 01:58:18.084101 9105 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:18.084393 9105 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:18.087268 9105 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_1'
I20240502 01:58:18.171520 8641 raft_consensus.cc:1261] T c49a55dc4dbf48da905d7cfa0bf2fb2a P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Refusing update from remote peer 1caeef741efa452fa9bb1fd3d47e65eb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:18.171603 8373 raft_consensus.cc:1261] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Refusing update from remote peer 1caeef741efa452fa9bb1fd3d47e65eb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:18.171598 8642 raft_consensus.cc:1261] T ed11274a6a2e4c07b272fe104533ea62 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Refusing update from remote peer 1caeef741efa452fa9bb1fd3d47e65eb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:18.172233 8995 consensus_queue.cc:1035] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [LEADER]: Connected to new peer: Peer: permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:18.172637 8995 consensus_queue.cc:1035] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [LEADER]: Connected to new peer: Peer: permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:18.172953 9098 consensus_queue.cc:1035] T c49a55dc4dbf48da905d7cfa0bf2fb2a P 1caeef741efa452fa9bb1fd3d47e65eb [LEADER]: Connected to new peer: Peer: permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:18.173316 8373 raft_consensus.cc:1261] T ed11274a6a2e4c07b272fe104533ea62 P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Refusing update from remote peer 1caeef741efa452fa9bb1fd3d47e65eb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:18.173650 9098 consensus_queue.cc:1035] T ed11274a6a2e4c07b272fe104533ea62 P 1caeef741efa452fa9bb1fd3d47e65eb [LEADER]: Connected to new peer: Peer: permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:18.182071 8507 raft_consensus.cc:1261] T 851d391e5df146f9858f5d07bf48a81b P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Refusing update from remote peer 32416365804d40db9d644b5d32301d54: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:18.182315 8240 raft_consensus.cc:1261] T 851d391e5df146f9858f5d07bf48a81b P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Refusing update from remote peer 32416365804d40db9d644b5d32301d54: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:18.182861 9102 consensus_queue.cc:1035] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:18.183331 9102 consensus_queue.cc:1035] T 851d391e5df146f9858f5d07bf48a81b P 32416365804d40db9d644b5d32301d54 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:18.197055 9056 mvcc.cc:205] Tried to move back new op lower bound from 7023063442104422400 to 7023063440893882368. Current Snapshot: MvccSnapshot[applied={T|T < 7023063442104422400}]
INSERT report
rows total: 2000
time total: 108.982 ms
time per row: 0.054491 ms
I20240502 01:58:18.245182 8095 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34536:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:18.251080 8220 tablet_service.cc:1467] Processing CreateTablet for tablet 54e81d5b68364c0494add244f66b0b6a (DEFAULT_TABLE table=rebalance_test_table_2 [id=7531060780f94ca2bbe3be44788d4574]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.251421 8220 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 54e81d5b68364c0494add244f66b0b6a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.251998 8757 tablet_service.cc:1467] Processing CreateTablet for tablet 54e81d5b68364c0494add244f66b0b6a (DEFAULT_TABLE table=rebalance_test_table_2 [id=7531060780f94ca2bbe3be44788d4574]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.252264 8757 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 54e81d5b68364c0494add244f66b0b6a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.252444 8485 tablet_service.cc:1467] Processing CreateTablet for tablet 54e81d5b68364c0494add244f66b0b6a (DEFAULT_TABLE table=rebalance_test_table_2 [id=7531060780f94ca2bbe3be44788d4574]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.252743 8485 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 54e81d5b68364c0494add244f66b0b6a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.254036 8219 tablet_service.cc:1467] Processing CreateTablet for tablet 2e289e488ac2434387d74ea429781bdb (DEFAULT_TABLE table=rebalance_test_table_2 [id=7531060780f94ca2bbe3be44788d4574]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.254273 8219 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e289e488ac2434387d74ea429781bdb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.254459 8994 tablet_bootstrap.cc:492] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:18.254638 8988 tablet_bootstrap.cc:492] T 54e81d5b68364c0494add244f66b0b6a P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:18.255118 8756 tablet_service.cc:1467] Processing CreateTablet for tablet 2e289e488ac2434387d74ea429781bdb (DEFAULT_TABLE table=rebalance_test_table_2 [id=7531060780f94ca2bbe3be44788d4574]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.255326 8756 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e289e488ac2434387d74ea429781bdb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.255429 8988 tablet_bootstrap.cc:654] T 54e81d5b68364c0494add244f66b0b6a P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.255467 8994 tablet_bootstrap.cc:654] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.252444 8486 tablet_service.cc:1467] Processing CreateTablet for tablet 2e289e488ac2434387d74ea429781bdb (DEFAULT_TABLE table=rebalance_test_table_2 [id=7531060780f94ca2bbe3be44788d4574]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.256130 8486 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e289e488ac2434387d74ea429781bdb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.257402 8985 tablet_bootstrap.cc:492] T 54e81d5b68364c0494add244f66b0b6a P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:18.257555 8352 tablet_service.cc:1467] Processing CreateTablet for tablet ede42eef861f40419854173443725c83 (DEFAULT_TABLE table=rebalance_test_table_2 [id=7531060780f94ca2bbe3be44788d4574]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.257836 8352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ede42eef861f40419854173443725c83. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.257891 8988 tablet_bootstrap.cc:492] T 54e81d5b68364c0494add244f66b0b6a P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:18.257974 8988 ts_tablet_manager.cc:1367] T 54e81d5b68364c0494add244f66b0b6a P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:18.258323 8985 tablet_bootstrap.cc:654] T 54e81d5b68364c0494add244f66b0b6a P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.258363 8988 raft_consensus.cc:348] T 54e81d5b68364c0494add244f66b0b6a P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.258441 8988 raft_consensus.cc:374] T 54e81d5b68364c0494add244f66b0b6a P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.258461 8988 raft_consensus.cc:729] T 54e81d5b68364c0494add244f66b0b6a P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.258569 8988 consensus_queue.cc:260] T 54e81d5b68364c0494add244f66b0b6a P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.258881 8988 ts_tablet_manager.cc:1398] T 54e81d5b68364c0494add244f66b0b6a P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:18.259047 8988 tablet_bootstrap.cc:492] T 2e289e488ac2434387d74ea429781bdb P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:18.259163 8891 tablet_service.cc:1467] Processing CreateTablet for tablet ede42eef861f40419854173443725c83 (DEFAULT_TABLE table=rebalance_test_table_2 [id=7531060780f94ca2bbe3be44788d4574]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.259449 8891 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ede42eef861f40419854173443725c83. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.259825 8988 tablet_bootstrap.cc:654] T 2e289e488ac2434387d74ea429781bdb P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.260171 8986 tablet_bootstrap.cc:492] T ede42eef861f40419854173443725c83 P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:18.261093 8986 tablet_bootstrap.cc:654] T ede42eef861f40419854173443725c83 P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.261678 8988 tablet_bootstrap.cc:492] T 2e289e488ac2434387d74ea429781bdb P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:18.261749 8988 ts_tablet_manager.cc:1367] T 2e289e488ac2434387d74ea429781bdb P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:18.261898 8621 tablet_service.cc:1467] Processing CreateTablet for tablet ede42eef861f40419854173443725c83 (DEFAULT_TABLE table=rebalance_test_table_2 [id=7531060780f94ca2bbe3be44788d4574]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.262209 8621 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ede42eef861f40419854173443725c83. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.262239 8988 raft_consensus.cc:348] T 2e289e488ac2434387d74ea429781bdb P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.262655 8986 tablet_bootstrap.cc:492] T ede42eef861f40419854173443725c83 P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:18.262349 8988 raft_consensus.cc:374] T 2e289e488ac2434387d74ea429781bdb P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.262742 8986 ts_tablet_manager.cc:1367] T ede42eef861f40419854173443725c83 P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:18.262753 8988 raft_consensus.cc:729] T 2e289e488ac2434387d74ea429781bdb P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.262802 8984 tablet_bootstrap.cc:492] T ede42eef861f40419854173443725c83 P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:18.262916 8988 consensus_queue.cc:260] T 2e289e488ac2434387d74ea429781bdb P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.263239 8986 raft_consensus.cc:348] T ede42eef861f40419854173443725c83 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.263324 8988 ts_tablet_manager.cc:1398] T 2e289e488ac2434387d74ea429781bdb P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:18.263341 8986 raft_consensus.cc:374] T ede42eef861f40419854173443725c83 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.263378 8986 raft_consensus.cc:729] T ede42eef861f40419854173443725c83 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.263500 8986 consensus_queue.cc:260] T ede42eef861f40419854173443725c83 P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.263691 8984 tablet_bootstrap.cc:654] T ede42eef861f40419854173443725c83 P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.263731 8986 ts_tablet_manager.cc:1398] T ede42eef861f40419854173443725c83 P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:18.268234 8987 tablet_bootstrap.cc:492] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:18.268604 8985 tablet_bootstrap.cc:492] T 54e81d5b68364c0494add244f66b0b6a P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:18.268697 8985 ts_tablet_manager.cc:1367] T 54e81d5b68364c0494add244f66b0b6a P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.002s
I20240502 01:58:18.269229 8987 tablet_bootstrap.cc:654] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.269140 8985 raft_consensus.cc:348] T 54e81d5b68364c0494add244f66b0b6a P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.269281 8985 raft_consensus.cc:374] T 54e81d5b68364c0494add244f66b0b6a P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.269318 8985 raft_consensus.cc:729] T 54e81d5b68364c0494add244f66b0b6a P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.269435 8985 consensus_queue.cc:260] T 54e81d5b68364c0494add244f66b0b6a P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.269825 8985 ts_tablet_manager.cc:1398] T 54e81d5b68364c0494add244f66b0b6a P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:18.269829 8984 tablet_bootstrap.cc:492] T ede42eef861f40419854173443725c83 P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:18.269915 8984 ts_tablet_manager.cc:1367] T ede42eef861f40419854173443725c83 P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 01:58:18.270037 8994 tablet_bootstrap.cc:492] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:18.270126 8994 ts_tablet_manager.cc:1367] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.016s user 0.002s sys 0.000s
I20240502 01:58:18.270388 8984 raft_consensus.cc:348] T ede42eef861f40419854173443725c83 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.270494 8984 raft_consensus.cc:374] T ede42eef861f40419854173443725c83 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.270527 8984 raft_consensus.cc:729] T ede42eef861f40419854173443725c83 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.270577 8985 tablet_bootstrap.cc:492] T 2e289e488ac2434387d74ea429781bdb P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:18.270632 8994 raft_consensus.cc:348] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.270742 8994 raft_consensus.cc:374] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.270696 8984 consensus_queue.cc:260] T ede42eef861f40419854173443725c83 P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.270774 8994 raft_consensus.cc:729] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.270896 8994 consensus_queue.cc:260] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.271211 8994 ts_tablet_manager.cc:1398] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:18.271381 8994 tablet_bootstrap.cc:492] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:18.271602 8987 tablet_bootstrap.cc:492] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:18.271688 8987 ts_tablet_manager.cc:1367] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:18.272141 8987 raft_consensus.cc:348] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.272238 8987 raft_consensus.cc:374] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.272271 8987 raft_consensus.cc:729] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.272419 8987 consensus_queue.cc:260] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.272678 8987 ts_tablet_manager.cc:1398] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:18.272969 8984 ts_tablet_manager.cc:1398] T ede42eef861f40419854173443725c83 P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:18.273144 8994 tablet_bootstrap.cc:654] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.273236 8985 tablet_bootstrap.cc:654] T 2e289e488ac2434387d74ea429781bdb P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.275061 8994 tablet_bootstrap.cc:492] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:18.275166 8994 ts_tablet_manager.cc:1367] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:58:18.275642 8994 raft_consensus.cc:348] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.275768 8994 raft_consensus.cc:374] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.275768 8985 tablet_bootstrap.cc:492] T 2e289e488ac2434387d74ea429781bdb P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:18.275806 8994 raft_consensus.cc:729] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.275956 8994 consensus_queue.cc:260] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.276265 8994 ts_tablet_manager.cc:1398] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:18.275871 8985 ts_tablet_manager.cc:1367] T 2e289e488ac2434387d74ea429781bdb P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.002s
I20240502 01:58:18.277118 8985 raft_consensus.cc:348] T 2e289e488ac2434387d74ea429781bdb P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.277236 8985 raft_consensus.cc:374] T 2e289e488ac2434387d74ea429781bdb P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.277272 8985 raft_consensus.cc:729] T 2e289e488ac2434387d74ea429781bdb P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.277437 8985 consensus_queue.cc:260] T 2e289e488ac2434387d74ea429781bdb P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.277786 8985 ts_tablet_manager.cc:1398] T 2e289e488ac2434387d74ea429781bdb P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:18.379096 9146 raft_consensus.cc:482] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:18.379221 9146 raft_consensus.cc:504] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.379660 9146 leader_election.cc:288] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e325acceaed541869d43c1f762530c2e (127.7.221.67:42267), e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919)
I20240502 01:58:18.380054 8506 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e289e488ac2434387d74ea429781bdb" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e" is_pre_election: true
I20240502 01:58:18.380052 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e289e488ac2434387d74ea429781bdb" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" is_pre_election: true
I20240502 01:58:18.380198 8506 raft_consensus.cc:2454] T 2e289e488ac2434387d74ea429781bdb P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 0.
I20240502 01:58:18.380198 8240 raft_consensus.cc:2454] T 2e289e488ac2434387d74ea429781bdb P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 0.
I20240502 01:58:18.380484 8709 leader_election.cc:302] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e325acceaed541869d43c1f762530c2e, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:18.380666 9146 raft_consensus.cc:2781] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:18.380735 9146 raft_consensus.cc:482] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:18.380764 9146 raft_consensus.cc:3037] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.381631 9146 raft_consensus.cc:504] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.381954 9146 leader_election.cc:288] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 election: Requested vote from peers e325acceaed541869d43c1f762530c2e (127.7.221.67:42267), e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919)
I20240502 01:58:18.382285 8506 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e289e488ac2434387d74ea429781bdb" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e"
I20240502 01:58:18.382397 8506 raft_consensus.cc:3037] T 2e289e488ac2434387d74ea429781bdb P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.383206 8506 raft_consensus.cc:2454] T 2e289e488ac2434387d74ea429781bdb P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 1.
I20240502 01:58:18.383493 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e289e488ac2434387d74ea429781bdb" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
I20240502 01:58:18.383549 8709 leader_election.cc:302] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e325acceaed541869d43c1f762530c2e; no voters:
I20240502 01:58:18.383646 8240 raft_consensus.cc:3037] T 2e289e488ac2434387d74ea429781bdb P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.383719 9146 raft_consensus.cc:2781] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:18.383828 9146 raft_consensus.cc:686] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [term 1 LEADER]: Becoming Leader. State: Replica: 32416365804d40db9d644b5d32301d54, State: Running, Role: LEADER
I20240502 01:58:18.384205 9146 consensus_queue.cc:237] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.384636 8240 raft_consensus.cc:2454] T 2e289e488ac2434387d74ea429781bdb P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 1.
I20240502 01:58:18.386495 8090 catalog_manager.cc:5484] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 reported cstate change: term changed from 0 to 1, leader changed from <none> to 32416365804d40db9d644b5d32301d54 (127.7.221.69). New cstate: current_term: 1 leader_uuid: "32416365804d40db9d644b5d32301d54" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: HEALTHY } } }
I20240502 01:58:18.454706 9000 raft_consensus.cc:482] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:18.454864 9000 raft_consensus.cc:504] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.455988 9000 leader_election.cc:288] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887), 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503)
I20240502 01:58:18.459720 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ede42eef861f40419854173443725c83" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" is_pre_election: true
I20240502 01:58:18.459918 8912 raft_consensus.cc:2454] T ede42eef861f40419854173443725c83 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:18.459919 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ede42eef861f40419854173443725c83" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15" is_pre_election: true
I20240502 01:58:18.460043 8373 raft_consensus.cc:2454] T ede42eef861f40419854173443725c83 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:18.460305 8573 leader_election.cc:302] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:18.460654 9000 raft_consensus.cc:2781] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:18.460703 9000 raft_consensus.cc:482] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:18.460731 9000 raft_consensus.cc:3037] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.461542 9000 raft_consensus.cc:504] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.461889 9000 leader_election.cc:288] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Requested vote from peers 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887), 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503)
I20240502 01:58:18.462226 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ede42eef861f40419854173443725c83" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15"
I20240502 01:58:18.462330 8373 raft_consensus.cc:3037] T ede42eef861f40419854173443725c83 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.463142 8373 raft_consensus.cc:2454] T ede42eef861f40419854173443725c83 P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:18.463402 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ede42eef861f40419854173443725c83" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
I20240502 01:58:18.463493 8574 leader_election.cc:302] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:18.463549 8912 raft_consensus.cc:3037] T ede42eef861f40419854173443725c83 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.463668 9000 raft_consensus.cc:2781] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:18.463829 9000 raft_consensus.cc:686] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 LEADER]: Becoming Leader. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Running, Role: LEADER
I20240502 01:58:18.464165 9000 consensus_queue.cc:237] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.464396 8912 raft_consensus.cc:2454] T ede42eef861f40419854173443725c83 P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:18.466506 8090 catalog_manager.cc:5484] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 reported cstate change: term changed from 0 to 1, leader changed from <none> to ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: HEALTHY } } }
I20240502 01:58:18.481359 9146 raft_consensus.cc:482] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:18.481482 9146 raft_consensus.cc:504] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.481909 9146 leader_election.cc:288] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), e325acceaed541869d43c1f762530c2e (127.7.221.67:42267)
I20240502 01:58:18.482389 8506 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "54e81d5b68364c0494add244f66b0b6a" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e" is_pre_election: true
I20240502 01:58:18.482405 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "54e81d5b68364c0494add244f66b0b6a" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" is_pre_election: true
I20240502 01:58:18.482537 8506 raft_consensus.cc:2454] T 54e81d5b68364c0494add244f66b0b6a P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 0.
I20240502 01:58:18.482550 8240 raft_consensus.cc:2454] T 54e81d5b68364c0494add244f66b0b6a P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 0.
I20240502 01:58:18.482933 8710 leader_election.cc:302] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:18.483237 9146 raft_consensus.cc:2781] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:18.483310 9146 raft_consensus.cc:482] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:18.483364 9146 raft_consensus.cc:3037] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.484160 9146 raft_consensus.cc:504] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.484503 9146 leader_election.cc:288] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 election: Requested vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), e325acceaed541869d43c1f762530c2e (127.7.221.67:42267)
I20240502 01:58:18.484861 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "54e81d5b68364c0494add244f66b0b6a" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
I20240502 01:58:18.484890 8506 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "54e81d5b68364c0494add244f66b0b6a" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e"
I20240502 01:58:18.484977 8240 raft_consensus.cc:3037] T 54e81d5b68364c0494add244f66b0b6a P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.484997 8506 raft_consensus.cc:3037] T 54e81d5b68364c0494add244f66b0b6a P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.485781 8240 raft_consensus.cc:2454] T 54e81d5b68364c0494add244f66b0b6a P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 1.
I20240502 01:58:18.485903 8506 raft_consensus.cc:2454] T 54e81d5b68364c0494add244f66b0b6a P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 1.
I20240502 01:58:18.486141 8710 leader_election.cc:302] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:18.486341 9146 raft_consensus.cc:2781] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:18.486414 9146 raft_consensus.cc:686] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [term 1 LEADER]: Becoming Leader. State: Replica: 32416365804d40db9d644b5d32301d54, State: Running, Role: LEADER
I20240502 01:58:18.486526 9146 consensus_queue.cc:237] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:18.488241 8090 catalog_manager.cc:5484] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 reported cstate change: term changed from 0 to 1, leader changed from <none> to 32416365804d40db9d644b5d32301d54 (127.7.221.69). New cstate: current_term: 1 leader_uuid: "32416365804d40db9d644b5d32301d54" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: HEALTHY } } }
W20240502 01:58:18.610999 9164 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:18.611230 9164 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:18.614154 9164 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_2'
I20240502 01:58:18.692835 8240 raft_consensus.cc:1261] T 2e289e488ac2434387d74ea429781bdb P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Refusing update from remote peer 32416365804d40db9d644b5d32301d54: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:18.693482 9102 consensus_queue.cc:1035] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:18.693954 8240 raft_consensus.cc:1261] T 54e81d5b68364c0494add244f66b0b6a P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Refusing update from remote peer 32416365804d40db9d644b5d32301d54: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:18.694268 8506 raft_consensus.cc:1261] T 2e289e488ac2434387d74ea429781bdb P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Refusing update from remote peer 32416365804d40db9d644b5d32301d54: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:18.694620 8507 raft_consensus.cc:1261] T 54e81d5b68364c0494add244f66b0b6a P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Refusing update from remote peer 32416365804d40db9d644b5d32301d54: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:18.695520 8373 raft_consensus.cc:1261] T ede42eef861f40419854173443725c83 P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Refusing update from remote peer ef9a99ad58b84fd2ada7f035f27361b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:18.695523 9146 consensus_queue.cc:1035] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:18.695976 9146 consensus_queue.cc:1035] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:18.696312 9146 consensus_queue.cc:1035] T 54e81d5b68364c0494add244f66b0b6a P 32416365804d40db9d644b5d32301d54 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:18.696645 9000 consensus_queue.cc:1035] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:18.702100 8912 raft_consensus.cc:1261] T ede42eef861f40419854173443725c83 P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Refusing update from remote peer ef9a99ad58b84fd2ada7f035f27361b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:18.706665 9000 consensus_queue.cc:1035] T ede42eef861f40419854173443725c83 P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 111.092 ms
time per row: 0.0555458 ms
I20240502 01:58:18.767472 8095 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34536:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:18.773010 8352 tablet_service.cc:1467] Processing CreateTablet for tablet 39d2476979754af48359409e8fa9554e (DEFAULT_TABLE table=rebalance_test_table_3 [id=572356d4638b48058e8e6476211e24ff]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.773133 8891 tablet_service.cc:1467] Processing CreateTablet for tablet 39d2476979754af48359409e8fa9554e (DEFAULT_TABLE table=rebalance_test_table_3 [id=572356d4638b48058e8e6476211e24ff]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.773231 8621 tablet_service.cc:1467] Processing CreateTablet for tablet 39d2476979754af48359409e8fa9554e (DEFAULT_TABLE table=rebalance_test_table_3 [id=572356d4638b48058e8e6476211e24ff]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.773387 8352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 39d2476979754af48359409e8fa9554e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.773434 8891 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 39d2476979754af48359409e8fa9554e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.773483 8621 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 39d2476979754af48359409e8fa9554e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.775146 8756 tablet_service.cc:1467] Processing CreateTablet for tablet 186a2b637e794290a08e62f8c513a5a0 (DEFAULT_TABLE table=rebalance_test_table_3 [id=572356d4638b48058e8e6476211e24ff]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.775391 8756 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 186a2b637e794290a08e62f8c513a5a0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.775619 9211 tablet_bootstrap.cc:492] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:18.776816 8621 tablet_service.cc:1467] Processing CreateTablet for tablet a776c692c47349c7bd02f016b14c5d0f (DEFAULT_TABLE table=rebalance_test_table_3 [id=572356d4638b48058e8e6476211e24ff]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.776947 9211 tablet_bootstrap.cc:654] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.777050 8621 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a776c692c47349c7bd02f016b14c5d0f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.777658 9212 tablet_bootstrap.cc:492] T 39d2476979754af48359409e8fa9554e P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:18.777940 8219 tablet_service.cc:1467] Processing CreateTablet for tablet 186a2b637e794290a08e62f8c513a5a0 (DEFAULT_TABLE table=rebalance_test_table_3 [id=572356d4638b48058e8e6476211e24ff]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.778244 8219 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 186a2b637e794290a08e62f8c513a5a0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.778770 8486 tablet_service.cc:1467] Processing CreateTablet for tablet 186a2b637e794290a08e62f8c513a5a0 (DEFAULT_TABLE table=rebalance_test_table_3 [id=572356d4638b48058e8e6476211e24ff]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.778990 8486 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 186a2b637e794290a08e62f8c513a5a0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.779050 9211 tablet_bootstrap.cc:492] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:18.779129 9211 ts_tablet_manager.cc:1367] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:58:18.779615 9211 raft_consensus.cc:348] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.779713 9211 raft_consensus.cc:374] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.779740 9211 raft_consensus.cc:729] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.779865 9211 consensus_queue.cc:260] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.780166 9211 ts_tablet_manager.cc:1398] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:18.780664 9212 tablet_bootstrap.cc:654] T 39d2476979754af48359409e8fa9554e P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.780601 8892 tablet_service.cc:1467] Processing CreateTablet for tablet a776c692c47349c7bd02f016b14c5d0f (DEFAULT_TABLE table=rebalance_test_table_3 [id=572356d4638b48058e8e6476211e24ff]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.780843 8892 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a776c692c47349c7bd02f016b14c5d0f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.781981 9211 tablet_bootstrap.cc:492] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:18.782392 9215 tablet_bootstrap.cc:492] T 186a2b637e794290a08e62f8c513a5a0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:18.782783 9212 tablet_bootstrap.cc:492] T 39d2476979754af48359409e8fa9554e P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:18.782867 9212 ts_tablet_manager.cc:1367] T 39d2476979754af48359409e8fa9554e P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 01:58:18.783759 9211 tablet_bootstrap.cc:654] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.783877 9212 raft_consensus.cc:348] T 39d2476979754af48359409e8fa9554e P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.784054 9212 raft_consensus.cc:374] T 39d2476979754af48359409e8fa9554e P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.784137 9212 raft_consensus.cc:729] T 39d2476979754af48359409e8fa9554e P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.784233 8352 tablet_service.cc:1467] Processing CreateTablet for tablet a776c692c47349c7bd02f016b14c5d0f (DEFAULT_TABLE table=rebalance_test_table_3 [id=572356d4638b48058e8e6476211e24ff]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:18.784317 9212 consensus_queue.cc:260] T 39d2476979754af48359409e8fa9554e P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.784478 8352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a776c692c47349c7bd02f016b14c5d0f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:18.784878 9215 tablet_bootstrap.cc:654] T 186a2b637e794290a08e62f8c513a5a0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.785027 9212 ts_tablet_manager.cc:1398] T 39d2476979754af48359409e8fa9554e P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:18.785792 9211 tablet_bootstrap.cc:492] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:18.785847 9211 ts_tablet_manager.cc:1367] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:58:18.786155 9211 raft_consensus.cc:348] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:18.786223 9211 raft_consensus.cc:374] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.786270 9211 raft_consensus.cc:729] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.786398 9211 consensus_queue.cc:260] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:18.787202 9211 ts_tablet_manager.cc:1398] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:18.787523 9212 tablet_bootstrap.cc:492] T a776c692c47349c7bd02f016b14c5d0f P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:18.788254 9215 tablet_bootstrap.cc:492] T 186a2b637e794290a08e62f8c513a5a0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:18.788343 9215 ts_tablet_manager.cc:1367] T 186a2b637e794290a08e62f8c513a5a0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 01:58:18.788437 9212 tablet_bootstrap.cc:654] T a776c692c47349c7bd02f016b14c5d0f P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.788763 9215 raft_consensus.cc:348] T 186a2b637e794290a08e62f8c513a5a0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:18.788841 9215 raft_consensus.cc:374] T 186a2b637e794290a08e62f8c513a5a0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.788820 9214 tablet_bootstrap.cc:492] T 186a2b637e794290a08e62f8c513a5a0 P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:18.788861 9215 raft_consensus.cc:729] T 186a2b637e794290a08e62f8c513a5a0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.788964 9215 consensus_queue.cc:260] T 186a2b637e794290a08e62f8c513a5a0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:18.789209 9215 ts_tablet_manager.cc:1398] T 186a2b637e794290a08e62f8c513a5a0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:18.789649 9214 tablet_bootstrap.cc:654] T 186a2b637e794290a08e62f8c513a5a0 P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.790381 9216 tablet_bootstrap.cc:492] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:18.790773 9220 tablet_bootstrap.cc:492] T a776c692c47349c7bd02f016b14c5d0f P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:18.790652 9212 tablet_bootstrap.cc:492] T a776c692c47349c7bd02f016b14c5d0f P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:18.791122 9212 ts_tablet_manager.cc:1367] T a776c692c47349c7bd02f016b14c5d0f P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:58:18.791536 9212 raft_consensus.cc:348] T a776c692c47349c7bd02f016b14c5d0f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:18.791656 9220 tablet_bootstrap.cc:654] T a776c692c47349c7bd02f016b14c5d0f P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.791666 9212 raft_consensus.cc:374] T a776c692c47349c7bd02f016b14c5d0f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.791700 9212 raft_consensus.cc:729] T a776c692c47349c7bd02f016b14c5d0f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.791864 9212 consensus_queue.cc:260] T a776c692c47349c7bd02f016b14c5d0f P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:18.792739 9212 ts_tablet_manager.cc:1398] T a776c692c47349c7bd02f016b14c5d0f P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:18.792860 9216 tablet_bootstrap.cc:654] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.793570 9214 tablet_bootstrap.cc:492] T 186a2b637e794290a08e62f8c513a5a0 P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:18.793653 9214 ts_tablet_manager.cc:1367] T 186a2b637e794290a08e62f8c513a5a0 P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 01:58:18.793707 9220 tablet_bootstrap.cc:492] T a776c692c47349c7bd02f016b14c5d0f P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:18.793799 9220 ts_tablet_manager.cc:1367] T a776c692c47349c7bd02f016b14c5d0f P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:18.794140 9214 raft_consensus.cc:348] T 186a2b637e794290a08e62f8c513a5a0 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:18.794279 9214 raft_consensus.cc:374] T 186a2b637e794290a08e62f8c513a5a0 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.794241 9220 raft_consensus.cc:348] T a776c692c47349c7bd02f016b14c5d0f P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:18.794338 9220 raft_consensus.cc:374] T a776c692c47349c7bd02f016b14c5d0f P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.794345 9214 raft_consensus.cc:729] T 186a2b637e794290a08e62f8c513a5a0 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.794474 9214 consensus_queue.cc:260] T 186a2b637e794290a08e62f8c513a5a0 P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:18.794795 9216 tablet_bootstrap.cc:492] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:18.794906 9216 ts_tablet_manager.cc:1367] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 01:58:18.794898 9214 ts_tablet_manager.cc:1398] T 186a2b637e794290a08e62f8c513a5a0 P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:18.795380 9216 raft_consensus.cc:348] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:18.795495 9216 raft_consensus.cc:374] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.795531 9216 raft_consensus.cc:729] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.795666 9216 consensus_queue.cc:260] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:18.795950 9216 ts_tablet_manager.cc:1398] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:18.796029 9220 raft_consensus.cc:729] T a776c692c47349c7bd02f016b14c5d0f P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.796175 9220 consensus_queue.cc:260] T a776c692c47349c7bd02f016b14c5d0f P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:18.796428 9220 ts_tablet_manager.cc:1398] T a776c692c47349c7bd02f016b14c5d0f P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:18.796581 9220 tablet_bootstrap.cc:492] T 39d2476979754af48359409e8fa9554e P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:18.797351 9220 tablet_bootstrap.cc:654] T 39d2476979754af48359409e8fa9554e P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:18.798352 9220 tablet_bootstrap.cc:492] T 39d2476979754af48359409e8fa9554e P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:18.798435 9220 ts_tablet_manager.cc:1367] T 39d2476979754af48359409e8fa9554e P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:18.798964 9220 raft_consensus.cc:348] T 39d2476979754af48359409e8fa9554e P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.799077 9220 raft_consensus.cc:374] T 39d2476979754af48359409e8fa9554e P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:18.799176 9220 raft_consensus.cc:729] T 39d2476979754af48359409e8fa9554e P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:18.799338 9220 consensus_queue.cc:260] T 39d2476979754af48359409e8fa9554e P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.799629 9220 ts_tablet_manager.cc:1398] T 39d2476979754af48359409e8fa9554e P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:18.841498 9000 raft_consensus.cc:482] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:18.841641 9000 raft_consensus.cc:504] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:18.842042 9000 leader_election.cc:288] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887), 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503)
I20240502 01:58:18.842425 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a776c692c47349c7bd02f016b14c5d0f" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" is_pre_election: true
I20240502 01:58:18.842425 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a776c692c47349c7bd02f016b14c5d0f" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15" is_pre_election: true
I20240502 01:58:18.842572 8373 raft_consensus.cc:2454] T a776c692c47349c7bd02f016b14c5d0f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:18.842559 8912 raft_consensus.cc:2454] T a776c692c47349c7bd02f016b14c5d0f P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:18.842923 8574 leader_election.cc:302] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:18.843114 9000 raft_consensus.cc:2781] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:18.843175 9000 raft_consensus.cc:482] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:18.843222 9000 raft_consensus.cc:3037] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.844066 9000 raft_consensus.cc:504] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:18.844398 9000 leader_election.cc:288] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Requested vote from peers 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887), 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503)
I20240502 01:58:18.844723 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a776c692c47349c7bd02f016b14c5d0f" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
I20240502 01:58:18.844743 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a776c692c47349c7bd02f016b14c5d0f" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15"
I20240502 01:58:18.844826 8912 raft_consensus.cc:3037] T a776c692c47349c7bd02f016b14c5d0f P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.844851 8373 raft_consensus.cc:3037] T a776c692c47349c7bd02f016b14c5d0f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.845589 8912 raft_consensus.cc:2454] T a776c692c47349c7bd02f016b14c5d0f P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:18.845649 8373 raft_consensus.cc:2454] T a776c692c47349c7bd02f016b14c5d0f P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:18.845871 8573 leader_election.cc:302] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:18.846026 9000 raft_consensus.cc:2781] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:18.846109 9000 raft_consensus.cc:686] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 LEADER]: Becoming Leader. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Running, Role: LEADER
I20240502 01:58:18.846247 9000 consensus_queue.cc:237] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:18.847839 8094 catalog_manager.cc:5484] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 reported cstate change: term changed from 0 to 1, leader changed from <none> to ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:18.864280 9000 raft_consensus.cc:482] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:18.864404 9000 raft_consensus.cc:504] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.864707 9000 leader_election.cc:288] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:18.865099 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "39d2476979754af48359409e8fa9554e" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15" is_pre_election: true
I20240502 01:58:18.865100 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "39d2476979754af48359409e8fa9554e" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" is_pre_election: true
I20240502 01:58:18.865216 8373 raft_consensus.cc:2454] T 39d2476979754af48359409e8fa9554e P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:18.865231 8912 raft_consensus.cc:2454] T 39d2476979754af48359409e8fa9554e P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:18.865509 8574 leader_election.cc:302] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:18.865691 9000 raft_consensus.cc:2781] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:18.865747 9000 raft_consensus.cc:482] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:18.865803 9000 raft_consensus.cc:3037] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.866494 9000 raft_consensus.cc:504] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.866828 9000 leader_election.cc:288] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Requested vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:18.867086 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "39d2476979754af48359409e8fa9554e" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15"
I20240502 01:58:18.867123 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "39d2476979754af48359409e8fa9554e" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
I20240502 01:58:18.867185 8373 raft_consensus.cc:3037] T 39d2476979754af48359409e8fa9554e P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.867218 8912 raft_consensus.cc:3037] T 39d2476979754af48359409e8fa9554e P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.867883 8912 raft_consensus.cc:2454] T 39d2476979754af48359409e8fa9554e P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:18.867887 8373 raft_consensus.cc:2454] T 39d2476979754af48359409e8fa9554e P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:18.868146 8573 leader_election.cc:302] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:18.868522 9000 raft_consensus.cc:2781] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:18.868647 9000 raft_consensus.cc:686] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 LEADER]: Becoming Leader. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Running, Role: LEADER
I20240502 01:58:18.868801 9000 consensus_queue.cc:237] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:18.870445 8094 catalog_manager.cc:5484] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 reported cstate change: term changed from 0 to 1, leader changed from <none> to ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: HEALTHY } } }
I20240502 01:58:18.973219 8996 raft_consensus.cc:482] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:18.973356 8996 raft_consensus.cc:504] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:18.973759 8996 leader_election.cc:288] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775), e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919)
I20240502 01:58:18.974237 8777 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "186a2b637e794290a08e62f8c513a5a0" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54" is_pre_election: true
I20240502 01:58:18.974237 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "186a2b637e794290a08e62f8c513a5a0" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" is_pre_election: true
I20240502 01:58:18.974414 8777 raft_consensus.cc:2454] T 186a2b637e794290a08e62f8c513a5a0 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 0.
I20240502 01:58:18.974414 8240 raft_consensus.cc:2454] T 186a2b637e794290a08e62f8c513a5a0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 0.
I20240502 01:58:18.974747 8441 leader_election.cc:302] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e325acceaed541869d43c1f762530c2e, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:18.974975 8996 raft_consensus.cc:2781] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:18.975047 8996 raft_consensus.cc:482] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:18.975075 8996 raft_consensus.cc:3037] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.975922 8996 raft_consensus.cc:504] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:18.976233 8996 leader_election.cc:288] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 election: Requested vote from peers 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775), e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919)
I20240502 01:58:18.976562 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "186a2b637e794290a08e62f8c513a5a0" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
I20240502 01:58:18.976598 8777 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "186a2b637e794290a08e62f8c513a5a0" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54"
I20240502 01:58:18.976673 8240 raft_consensus.cc:3037] T 186a2b637e794290a08e62f8c513a5a0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.976730 8777 raft_consensus.cc:3037] T 186a2b637e794290a08e62f8c513a5a0 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:18.977478 8240 raft_consensus.cc:2454] T 186a2b637e794290a08e62f8c513a5a0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 1.
I20240502 01:58:18.977573 8777 raft_consensus.cc:2454] T 186a2b637e794290a08e62f8c513a5a0 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 1.
I20240502 01:58:18.977764 8441 leader_election.cc:302] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e325acceaed541869d43c1f762530c2e, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:18.977924 8996 raft_consensus.cc:2781] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:18.978104 8996 raft_consensus.cc:686] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [term 1 LEADER]: Becoming Leader. State: Replica: e325acceaed541869d43c1f762530c2e, State: Running, Role: LEADER
I20240502 01:58:18.978242 8996 consensus_queue.cc:237] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:18.980021 8094 catalog_manager.cc:5484] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e reported cstate change: term changed from 0 to 1, leader changed from <none> to e325acceaed541869d43c1f762530c2e (127.7.221.67). New cstate: current_term: 1 leader_uuid: "e325acceaed541869d43c1f762530c2e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: HEALTHY } } }
W20240502 01:58:19.119596 9230 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:19.119843 9230 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:19.123178 9230 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_3'
I20240502 01:58:19.202198 8912 raft_consensus.cc:1261] T 39d2476979754af48359409e8fa9554e P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Refusing update from remote peer ef9a99ad58b84fd2ada7f035f27361b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:19.202301 8373 raft_consensus.cc:1261] T 39d2476979754af48359409e8fa9554e P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Refusing update from remote peer ef9a99ad58b84fd2ada7f035f27361b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:19.202797 9161 consensus_queue.cc:1035] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:19.202955 9000 consensus_queue.cc:1035] T 39d2476979754af48359409e8fa9554e P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:19.206662 8373 raft_consensus.cc:1261] T a776c692c47349c7bd02f016b14c5d0f P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Refusing update from remote peer ef9a99ad58b84fd2ada7f035f27361b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:19.212435 8240 raft_consensus.cc:1261] T 186a2b637e794290a08e62f8c513a5a0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Refusing update from remote peer e325acceaed541869d43c1f762530c2e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:19.213183 8777 raft_consensus.cc:1261] T 186a2b637e794290a08e62f8c513a5a0 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Refusing update from remote peer e325acceaed541869d43c1f762530c2e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:19.213663 9228 consensus_queue.cc:1035] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [LEADER]: Connected to new peer: Peer: permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:19.213832 8996 consensus_queue.cc:1035] T 186a2b637e794290a08e62f8c513a5a0 P e325acceaed541869d43c1f762530c2e [LEADER]: Connected to new peer: Peer: permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:19.216964 9161 consensus_queue.cc:1035] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:19.218109 8911 raft_consensus.cc:1261] T a776c692c47349c7bd02f016b14c5d0f P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Refusing update from remote peer ef9a99ad58b84fd2ada7f035f27361b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:19.230120 9161 consensus_queue.cc:1035] T a776c692c47349c7bd02f016b14c5d0f P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 122.515 ms
time per row: 0.0612577 ms
I20240502 01:58:19.286938 8092 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34536:
name: "rebalance_test_table_4"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:19.293490 8219 tablet_service.cc:1467] Processing CreateTablet for tablet 99253785df6242b3bc13ca644998b7f9 (DEFAULT_TABLE table=rebalance_test_table_4 [id=743dfa1b6bf145e2b0e0ae1db5c19ed0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.293859 8219 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 99253785df6242b3bc13ca644998b7f9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.293792 8891 tablet_service.cc:1467] Processing CreateTablet for tablet 05cfe76c4e64414d85ff9a48d2cf1beb (DEFAULT_TABLE table=rebalance_test_table_4 [id=743dfa1b6bf145e2b0e0ae1db5c19ed0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.294065 8891 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 05cfe76c4e64414d85ff9a48d2cf1beb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.294682 8756 tablet_service.cc:1467] Processing CreateTablet for tablet 99253785df6242b3bc13ca644998b7f9 (DEFAULT_TABLE table=rebalance_test_table_4 [id=743dfa1b6bf145e2b0e0ae1db5c19ed0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.294981 8756 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 99253785df6242b3bc13ca644998b7f9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.295423 8621 tablet_service.cc:1467] Processing CreateTablet for tablet 05cfe76c4e64414d85ff9a48d2cf1beb (DEFAULT_TABLE table=rebalance_test_table_4 [id=743dfa1b6bf145e2b0e0ae1db5c19ed0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.295717 8621 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 05cfe76c4e64414d85ff9a48d2cf1beb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.297166 8220 tablet_service.cc:1467] Processing CreateTablet for tablet bd1053bf4d774faab67bd6a7fdf03eb0 (DEFAULT_TABLE table=rebalance_test_table_4 [id=743dfa1b6bf145e2b0e0ae1db5c19ed0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.297356 8220 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd1053bf4d774faab67bd6a7fdf03eb0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.298022 9273 tablet_bootstrap.cc:492] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:19.299060 9214 tablet_bootstrap.cc:492] T 99253785df6242b3bc13ca644998b7f9 P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:19.299207 9273 tablet_bootstrap.cc:654] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.299875 9214 tablet_bootstrap.cc:654] T 99253785df6242b3bc13ca644998b7f9 P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.300132 9220 tablet_bootstrap.cc:492] T 05cfe76c4e64414d85ff9a48d2cf1beb P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:19.301343 9220 tablet_bootstrap.cc:654] T 05cfe76c4e64414d85ff9a48d2cf1beb P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.301647 9273 tablet_bootstrap.cc:492] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:19.301781 9273 ts_tablet_manager.cc:1367] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:58:19.302088 8486 tablet_service.cc:1467] Processing CreateTablet for tablet 99253785df6242b3bc13ca644998b7f9 (DEFAULT_TABLE table=rebalance_test_table_4 [id=743dfa1b6bf145e2b0e0ae1db5c19ed0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.302356 8486 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 99253785df6242b3bc13ca644998b7f9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.302321 9273 raft_consensus.cc:348] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.302451 9273 raft_consensus.cc:374] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.302486 9273 raft_consensus.cc:729] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.302700 9273 consensus_queue.cc:260] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.302909 9214 tablet_bootstrap.cc:492] T 99253785df6242b3bc13ca644998b7f9 P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:19.302992 9214 ts_tablet_manager.cc:1367] T 99253785df6242b3bc13ca644998b7f9 P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:58:19.303464 9273 ts_tablet_manager.cc:1398] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 01:58:19.303601 9214 raft_consensus.cc:348] T 99253785df6242b3bc13ca644998b7f9 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.303710 9214 raft_consensus.cc:374] T 99253785df6242b3bc13ca644998b7f9 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.303740 9214 raft_consensus.cc:729] T 99253785df6242b3bc13ca644998b7f9 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.303867 9214 consensus_queue.cc:260] T 99253785df6242b3bc13ca644998b7f9 P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.304123 9214 ts_tablet_manager.cc:1398] T 99253785df6242b3bc13ca644998b7f9 P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:19.305299 8352 tablet_service.cc:1467] Processing CreateTablet for tablet 05cfe76c4e64414d85ff9a48d2cf1beb (DEFAULT_TABLE table=rebalance_test_table_4 [id=743dfa1b6bf145e2b0e0ae1db5c19ed0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.305608 8352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 05cfe76c4e64414d85ff9a48d2cf1beb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.307464 8485 tablet_service.cc:1467] Processing CreateTablet for tablet bd1053bf4d774faab67bd6a7fdf03eb0 (DEFAULT_TABLE table=rebalance_test_table_4 [id=743dfa1b6bf145e2b0e0ae1db5c19ed0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.308132 9277 tablet_bootstrap.cc:492] T 05cfe76c4e64414d85ff9a48d2cf1beb P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:19.309067 9277 tablet_bootstrap.cc:654] T 05cfe76c4e64414d85ff9a48d2cf1beb P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.309154 8485 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd1053bf4d774faab67bd6a7fdf03eb0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.310199 9277 tablet_bootstrap.cc:492] T 05cfe76c4e64414d85ff9a48d2cf1beb P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:19.310178 9279 tablet_bootstrap.cc:492] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:19.310338 9277 ts_tablet_manager.cc:1367] T 05cfe76c4e64414d85ff9a48d2cf1beb P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:19.310817 9277 raft_consensus.cc:348] T 05cfe76c4e64414d85ff9a48d2cf1beb P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.310992 9277 raft_consensus.cc:374] T 05cfe76c4e64414d85ff9a48d2cf1beb P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.311090 9277 raft_consensus.cc:729] T 05cfe76c4e64414d85ff9a48d2cf1beb P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.311252 9277 consensus_queue.cc:260] T 05cfe76c4e64414d85ff9a48d2cf1beb P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.311504 9220 tablet_bootstrap.cc:492] T 05cfe76c4e64414d85ff9a48d2cf1beb P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:19.311618 9220 ts_tablet_manager.cc:1367] T 05cfe76c4e64414d85ff9a48d2cf1beb P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.012s user 0.002s sys 0.000s
I20240502 01:58:19.311753 9277 ts_tablet_manager.cc:1398] T 05cfe76c4e64414d85ff9a48d2cf1beb P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:19.312114 9220 raft_consensus.cc:348] T 05cfe76c4e64414d85ff9a48d2cf1beb P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.312228 9220 raft_consensus.cc:374] T 05cfe76c4e64414d85ff9a48d2cf1beb P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.312263 9220 raft_consensus.cc:729] T 05cfe76c4e64414d85ff9a48d2cf1beb P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.312408 9220 consensus_queue.cc:260] T 05cfe76c4e64414d85ff9a48d2cf1beb P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.312727 9220 ts_tablet_manager.cc:1398] T 05cfe76c4e64414d85ff9a48d2cf1beb P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:19.312767 9279 tablet_bootstrap.cc:654] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.313906 9259 raft_consensus.cc:482] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:19.313993 9259 raft_consensus.cc:504] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.314568 8372 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "05cfe76c4e64414d85ff9a48d2cf1beb" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15" is_pre_election: true
I20240502 01:58:19.314721 8372 raft_consensus.cc:2454] T 05cfe76c4e64414d85ff9a48d2cf1beb P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:19.315147 8574 leader_election.cc:302] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:19.315326 9000 raft_consensus.cc:2781] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:19.315369 9000 raft_consensus.cc:482] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:19.315412 9000 raft_consensus.cc:3037] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.315605 9279 tablet_bootstrap.cc:492] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:19.315739 9279 ts_tablet_manager.cc:1367] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 01:58:19.316200 8911 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "05cfe76c4e64414d85ff9a48d2cf1beb" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" is_pre_election: true
I20240502 01:58:19.316346 8911 raft_consensus.cc:2454] T 05cfe76c4e64414d85ff9a48d2cf1beb P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:19.316344 9259 leader_election.cc:288] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:19.316368 9000 raft_consensus.cc:504] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.316723 9000 leader_election.cc:288] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Requested vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:19.317137 8911 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "05cfe76c4e64414d85ff9a48d2cf1beb" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
I20240502 01:58:19.317236 8911 raft_consensus.cc:3037] T 05cfe76c4e64414d85ff9a48d2cf1beb P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.317282 8372 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "05cfe76c4e64414d85ff9a48d2cf1beb" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15"
I20240502 01:58:19.317368 8372 raft_consensus.cc:3037] T 05cfe76c4e64414d85ff9a48d2cf1beb P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.317950 8911 raft_consensus.cc:2454] T 05cfe76c4e64414d85ff9a48d2cf1beb P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:19.318162 8372 raft_consensus.cc:2454] T 05cfe76c4e64414d85ff9a48d2cf1beb P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:19.318181 8573 leader_election.cc:302] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:19.318315 9000 raft_consensus.cc:2781] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:19.318388 9000 raft_consensus.cc:686] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 LEADER]: Becoming Leader. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Running, Role: LEADER
I20240502 01:58:19.318508 9000 consensus_queue.cc:237] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.318819 9279 raft_consensus.cc:348] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.318928 9279 raft_consensus.cc:374] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.318956 9279 raft_consensus.cc:729] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.319068 9279 consensus_queue.cc:260] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.319417 9279 ts_tablet_manager.cc:1398] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.003s user 0.000s sys 0.001s
I20240502 01:58:19.319607 9279 tablet_bootstrap.cc:492] T bd1053bf4d774faab67bd6a7fdf03eb0 P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:19.320248 8092 catalog_manager.cc:5484] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 reported cstate change: term changed from 0 to 1, leader changed from <none> to ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:19.320627 9279 tablet_bootstrap.cc:654] T bd1053bf4d774faab67bd6a7fdf03eb0 P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.321514 8756 tablet_service.cc:1467] Processing CreateTablet for tablet bd1053bf4d774faab67bd6a7fdf03eb0 (DEFAULT_TABLE table=rebalance_test_table_4 [id=743dfa1b6bf145e2b0e0ae1db5c19ed0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.321825 8756 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd1053bf4d774faab67bd6a7fdf03eb0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.322664 9279 tablet_bootstrap.cc:492] T bd1053bf4d774faab67bd6a7fdf03eb0 P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:19.322974 9279 ts_tablet_manager.cc:1367] T bd1053bf4d774faab67bd6a7fdf03eb0 P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.001s
I20240502 01:58:19.323307 9279 raft_consensus.cc:348] T bd1053bf4d774faab67bd6a7fdf03eb0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.323423 9279 raft_consensus.cc:374] T bd1053bf4d774faab67bd6a7fdf03eb0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.323449 9279 raft_consensus.cc:729] T bd1053bf4d774faab67bd6a7fdf03eb0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.323592 9279 consensus_queue.cc:260] T bd1053bf4d774faab67bd6a7fdf03eb0 P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.323828 9279 ts_tablet_manager.cc:1398] T bd1053bf4d774faab67bd6a7fdf03eb0 P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:19.324129 9214 tablet_bootstrap.cc:492] T bd1053bf4d774faab67bd6a7fdf03eb0 P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:19.324916 9214 tablet_bootstrap.cc:654] T bd1053bf4d774faab67bd6a7fdf03eb0 P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.330912 9272 tablet_bootstrap.cc:492] T 99253785df6242b3bc13ca644998b7f9 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:19.332057 9272 tablet_bootstrap.cc:654] T 99253785df6242b3bc13ca644998b7f9 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.334012 9272 tablet_bootstrap.cc:492] T 99253785df6242b3bc13ca644998b7f9 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:19.334092 9214 tablet_bootstrap.cc:492] T bd1053bf4d774faab67bd6a7fdf03eb0 P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:19.334175 9272 ts_tablet_manager.cc:1367] T 99253785df6242b3bc13ca644998b7f9 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:19.334234 9214 ts_tablet_manager.cc:1367] T bd1053bf4d774faab67bd6a7fdf03eb0 P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.000s
I20240502 01:58:19.334791 9214 raft_consensus.cc:348] T bd1053bf4d774faab67bd6a7fdf03eb0 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.334898 9214 raft_consensus.cc:374] T bd1053bf4d774faab67bd6a7fdf03eb0 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.334928 9214 raft_consensus.cc:729] T bd1053bf4d774faab67bd6a7fdf03eb0 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.335054 9214 consensus_queue.cc:260] T bd1053bf4d774faab67bd6a7fdf03eb0 P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.335060 9272 raft_consensus.cc:348] T 99253785df6242b3bc13ca644998b7f9 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.335156 9272 raft_consensus.cc:374] T 99253785df6242b3bc13ca644998b7f9 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.335180 9272 raft_consensus.cc:729] T 99253785df6242b3bc13ca644998b7f9 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.335323 9214 ts_tablet_manager.cc:1398] T bd1053bf4d774faab67bd6a7fdf03eb0 P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:19.335307 9272 consensus_queue.cc:260] T 99253785df6242b3bc13ca644998b7f9 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.335620 9272 ts_tablet_manager.cc:1398] T 99253785df6242b3bc13ca644998b7f9 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:19.338790 9272 tablet_bootstrap.cc:492] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:19.340282 9272 tablet_bootstrap.cc:654] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.341522 9272 tablet_bootstrap.cc:492] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:19.341642 9272 ts_tablet_manager.cc:1367] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:19.342124 9272 raft_consensus.cc:348] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.342226 9272 raft_consensus.cc:374] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.342257 9272 raft_consensus.cc:729] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.342406 9272 consensus_queue.cc:260] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.342748 9272 ts_tablet_manager.cc:1398] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:19.497620 8996 raft_consensus.cc:482] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:19.497737 8996 raft_consensus.cc:504] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.498138 8996 leader_election.cc:288] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775)
I20240502 01:58:19.498565 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99253785df6242b3bc13ca644998b7f9" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" is_pre_election: true
I20240502 01:58:19.498580 8777 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99253785df6242b3bc13ca644998b7f9" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54" is_pre_election: true
I20240502 01:58:19.498756 8240 raft_consensus.cc:2454] T 99253785df6242b3bc13ca644998b7f9 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 0.
I20240502 01:58:19.498766 8777 raft_consensus.cc:2454] T 99253785df6242b3bc13ca644998b7f9 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 0.
I20240502 01:58:19.499074 8439 leader_election.cc:302] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e325acceaed541869d43c1f762530c2e; no voters:
I20240502 01:58:19.499320 8996 raft_consensus.cc:2781] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:19.499374 8996 raft_consensus.cc:482] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:19.499415 8996 raft_consensus.cc:3037] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.500181 8996 raft_consensus.cc:504] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.500499 8996 leader_election.cc:288] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 election: Requested vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775)
I20240502 01:58:19.500818 8777 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99253785df6242b3bc13ca644998b7f9" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54"
I20240502 01:58:19.500810 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99253785df6242b3bc13ca644998b7f9" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
I20240502 01:58:19.500933 8240 raft_consensus.cc:3037] T 99253785df6242b3bc13ca644998b7f9 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.500933 8777 raft_consensus.cc:3037] T 99253785df6242b3bc13ca644998b7f9 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.501834 8240 raft_consensus.cc:2454] T 99253785df6242b3bc13ca644998b7f9 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 1.
I20240502 01:58:19.501822 8777 raft_consensus.cc:2454] T 99253785df6242b3bc13ca644998b7f9 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 1.
I20240502 01:58:19.502137 8439 leader_election.cc:302] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e325acceaed541869d43c1f762530c2e; no voters:
I20240502 01:58:19.502279 8996 raft_consensus.cc:2781] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:19.502367 8996 raft_consensus.cc:686] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [term 1 LEADER]: Becoming Leader. State: Replica: e325acceaed541869d43c1f762530c2e, State: Running, Role: LEADER
I20240502 01:58:19.502511 8996 consensus_queue.cc:237] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.504424 8092 catalog_manager.cc:5484] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e reported cstate change: term changed from 0 to 1, leader changed from <none> to e325acceaed541869d43c1f762530c2e (127.7.221.67). New cstate: current_term: 1 leader_uuid: "e325acceaed541869d43c1f762530c2e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:19.518288 8997 raft_consensus.cc:482] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:19.518400 8997 raft_consensus.cc:504] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.518822 8997 leader_election.cc:288] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e325acceaed541869d43c1f762530c2e (127.7.221.67:42267), 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775)
I20240502 01:58:19.519286 8507 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd1053bf4d774faab67bd6a7fdf03eb0" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e" is_pre_election: true
I20240502 01:58:19.519455 8507 raft_consensus.cc:2454] T bd1053bf4d774faab67bd6a7fdf03eb0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 0.
I20240502 01:58:19.519752 8173 leader_election.cc:302] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e325acceaed541869d43c1f762530c2e, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:19.519922 8997 raft_consensus.cc:2781] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:19.519974 8997 raft_consensus.cc:482] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:19.520015 8997 raft_consensus.cc:3037] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.520913 8997 raft_consensus.cc:504] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.521292 8997 leader_election.cc:288] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 election: Requested vote from peers e325acceaed541869d43c1f762530c2e (127.7.221.67:42267), 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775)
I20240502 01:58:19.521613 8507 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd1053bf4d774faab67bd6a7fdf03eb0" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e"
I20240502 01:58:19.521744 8507 raft_consensus.cc:3037] T bd1053bf4d774faab67bd6a7fdf03eb0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.522696 8507 raft_consensus.cc:2454] T bd1053bf4d774faab67bd6a7fdf03eb0 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 1.
I20240502 01:58:19.522828 8776 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd1053bf4d774faab67bd6a7fdf03eb0" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54"
I20240502 01:58:19.522826 8777 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd1053bf4d774faab67bd6a7fdf03eb0" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54" is_pre_election: true
I20240502 01:58:19.523016 8776 raft_consensus.cc:3037] T bd1053bf4d774faab67bd6a7fdf03eb0 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.523000 8173 leader_election.cc:302] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e325acceaed541869d43c1f762530c2e, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:19.523151 8997 raft_consensus.cc:2781] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:19.523311 8997 raft_consensus.cc:686] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 LEADER]: Becoming Leader. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Running, Role: LEADER
I20240502 01:58:19.523437 8997 consensus_queue.cc:237] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.523887 8776 raft_consensus.cc:2454] T bd1053bf4d774faab67bd6a7fdf03eb0 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 1.
I20240502 01:58:19.525148 8092 catalog_manager.cc:5484] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 reported cstate change: term changed from 0 to 1, leader changed from <none> to e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:19.648033 9291 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:19.648270 9291 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:19.651497 9291 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_4'
I20240502 01:58:19.740562 8911 raft_consensus.cc:1261] T 05cfe76c4e64414d85ff9a48d2cf1beb P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Refusing update from remote peer ef9a99ad58b84fd2ada7f035f27361b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:19.741586 8372 raft_consensus.cc:1261] T 05cfe76c4e64414d85ff9a48d2cf1beb P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Refusing update from remote peer ef9a99ad58b84fd2ada7f035f27361b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:19.742044 8776 raft_consensus.cc:1261] T 99253785df6242b3bc13ca644998b7f9 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Refusing update from remote peer e325acceaed541869d43c1f762530c2e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:19.742144 9000 consensus_queue.cc:1035] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:19.742895 9267 consensus_queue.cc:1035] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [LEADER]: Connected to new peer: Peer: permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:19.743453 9161 consensus_queue.cc:1035] T 05cfe76c4e64414d85ff9a48d2cf1beb P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:19.743711 8240 raft_consensus.cc:1261] T 99253785df6242b3bc13ca644998b7f9 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Refusing update from remote peer e325acceaed541869d43c1f762530c2e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:19.744117 9267 consensus_queue.cc:1035] T 99253785df6242b3bc13ca644998b7f9 P e325acceaed541869d43c1f762530c2e [LEADER]: Connected to new peer: Peer: permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:19.744699 8777 raft_consensus.cc:1261] T bd1053bf4d774faab67bd6a7fdf03eb0 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Refusing update from remote peer e7a8991bb9ff47489bf8ccbaf5ca7fa9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:19.747643 9289 consensus_queue.cc:1035] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:19.750104 8507 raft_consensus.cc:1261] T bd1053bf4d774faab67bd6a7fdf03eb0 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Refusing update from remote peer e7a8991bb9ff47489bf8ccbaf5ca7fa9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:19.750509 8997 consensus_queue.cc:1035] T bd1053bf4d774faab67bd6a7fdf03eb0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 111.934 ms
time per row: 0.0559671 ms
I20240502 01:58:19.806830 8093 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34536:
name: "rebalance_test_table_5"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:19.813334 8621 tablet_service.cc:1467] Processing CreateTablet for tablet 10e9d48d98ca4da286c329bf76028c05 (DEFAULT_TABLE table=rebalance_test_table_5 [id=0ad6d027fa6d45fd95f16593130b5d21]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.813684 8621 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 10e9d48d98ca4da286c329bf76028c05. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.813714 8352 tablet_service.cc:1467] Processing CreateTablet for tablet 10e9d48d98ca4da286c329bf76028c05 (DEFAULT_TABLE table=rebalance_test_table_5 [id=0ad6d027fa6d45fd95f16593130b5d21]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.813840 8220 tablet_service.cc:1467] Processing CreateTablet for tablet b90c8cce99b94418bf2a4b511cf38e49 (DEFAULT_TABLE table=rebalance_test_table_5 [id=0ad6d027fa6d45fd95f16593130b5d21]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.813956 8352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 10e9d48d98ca4da286c329bf76028c05. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.814252 8220 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b90c8cce99b94418bf2a4b511cf38e49. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.816087 9272 tablet_bootstrap.cc:492] T b90c8cce99b94418bf2a4b511cf38e49 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:19.816210 9344 tablet_bootstrap.cc:492] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:19.816267 8891 tablet_service.cc:1467] Processing CreateTablet for tablet 10e9d48d98ca4da286c329bf76028c05 (DEFAULT_TABLE table=rebalance_test_table_5 [id=0ad6d027fa6d45fd95f16593130b5d21]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.816550 8891 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 10e9d48d98ca4da286c329bf76028c05. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.817000 9344 tablet_bootstrap.cc:654] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.816994 9272 tablet_bootstrap.cc:654] T b90c8cce99b94418bf2a4b511cf38e49 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.818406 8892 tablet_service.cc:1467] Processing CreateTablet for tablet 58f576b66ff94cf988d96c2be32232aa (DEFAULT_TABLE table=rebalance_test_table_5 [id=0ad6d027fa6d45fd95f16593130b5d21]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.818571 8756 tablet_service.cc:1467] Processing CreateTablet for tablet b90c8cce99b94418bf2a4b511cf38e49 (DEFAULT_TABLE table=rebalance_test_table_5 [id=0ad6d027fa6d45fd95f16593130b5d21]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.818657 8892 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 58f576b66ff94cf988d96c2be32232aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.818851 8756 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b90c8cce99b94418bf2a4b511cf38e49. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.818907 8485 tablet_service.cc:1467] Processing CreateTablet for tablet b90c8cce99b94418bf2a4b511cf38e49 (DEFAULT_TABLE table=rebalance_test_table_5 [id=0ad6d027fa6d45fd95f16593130b5d21]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.819070 9272 tablet_bootstrap.cc:492] T b90c8cce99b94418bf2a4b511cf38e49 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:19.819164 8485 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b90c8cce99b94418bf2a4b511cf38e49. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.819202 9272 ts_tablet_manager.cc:1367] T b90c8cce99b94418bf2a4b511cf38e49 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.001s
I20240502 01:58:19.819816 9272 raft_consensus.cc:348] T b90c8cce99b94418bf2a4b511cf38e49 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.819924 9272 raft_consensus.cc:374] T b90c8cce99b94418bf2a4b511cf38e49 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.819973 9272 raft_consensus.cc:729] T b90c8cce99b94418bf2a4b511cf38e49 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.820093 9272 consensus_queue.cc:260] T b90c8cce99b94418bf2a4b511cf38e49 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.820346 9272 ts_tablet_manager.cc:1398] T b90c8cce99b94418bf2a4b511cf38e49 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:19.821172 9214 tablet_bootstrap.cc:492] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:19.821290 9346 tablet_bootstrap.cc:492] T 10e9d48d98ca4da286c329bf76028c05 P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:19.821538 9279 tablet_bootstrap.cc:492] T b90c8cce99b94418bf2a4b511cf38e49 P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:19.821904 9214 tablet_bootstrap.cc:654] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.822170 9346 tablet_bootstrap.cc:654] T 10e9d48d98ca4da286c329bf76028c05 P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.822482 9279 tablet_bootstrap.cc:654] T b90c8cce99b94418bf2a4b511cf38e49 P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.823134 8352 tablet_service.cc:1467] Processing CreateTablet for tablet 58f576b66ff94cf988d96c2be32232aa (DEFAULT_TABLE table=rebalance_test_table_5 [id=0ad6d027fa6d45fd95f16593130b5d21]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.823348 8352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 58f576b66ff94cf988d96c2be32232aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.823874 9346 tablet_bootstrap.cc:492] T 10e9d48d98ca4da286c329bf76028c05 P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:19.823959 9346 ts_tablet_manager.cc:1367] T 10e9d48d98ca4da286c329bf76028c05 P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:19.824002 9214 tablet_bootstrap.cc:492] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:19.824110 9214 ts_tablet_manager.cc:1367] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:19.824447 9346 raft_consensus.cc:348] T 10e9d48d98ca4da286c329bf76028c05 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.824524 9214 raft_consensus.cc:348] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.824627 9214 raft_consensus.cc:374] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.824640 9346 raft_consensus.cc:374] T 10e9d48d98ca4da286c329bf76028c05 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.824659 9214 raft_consensus.cc:729] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.824680 9346 raft_consensus.cc:729] T 10e9d48d98ca4da286c329bf76028c05 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.824774 9214 consensus_queue.cc:260] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.824831 9346 consensus_queue.cc:260] T 10e9d48d98ca4da286c329bf76028c05 P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.825066 9214 ts_tablet_manager.cc:1398] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:19.825105 9346 ts_tablet_manager.cc:1398] T 10e9d48d98ca4da286c329bf76028c05 P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:19.825712 8621 tablet_service.cc:1467] Processing CreateTablet for tablet 58f576b66ff94cf988d96c2be32232aa (DEFAULT_TABLE table=rebalance_test_table_5 [id=0ad6d027fa6d45fd95f16593130b5d21]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:19.825991 8621 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 58f576b66ff94cf988d96c2be32232aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:19.826830 9346 tablet_bootstrap.cc:492] T 58f576b66ff94cf988d96c2be32232aa P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:19.827797 9346 tablet_bootstrap.cc:654] T 58f576b66ff94cf988d96c2be32232aa P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.828624 9349 tablet_bootstrap.cc:492] T 58f576b66ff94cf988d96c2be32232aa P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:19.829016 9346 tablet_bootstrap.cc:492] T 58f576b66ff94cf988d96c2be32232aa P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:19.829087 9346 ts_tablet_manager.cc:1367] T 58f576b66ff94cf988d96c2be32232aa P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:19.829444 9349 tablet_bootstrap.cc:654] T 58f576b66ff94cf988d96c2be32232aa P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.829490 9346 raft_consensus.cc:348] T 58f576b66ff94cf988d96c2be32232aa P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.829603 9346 raft_consensus.cc:374] T 58f576b66ff94cf988d96c2be32232aa P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.829672 9346 raft_consensus.cc:729] T 58f576b66ff94cf988d96c2be32232aa P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.829818 9346 consensus_queue.cc:260] T 58f576b66ff94cf988d96c2be32232aa P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.830212 9346 ts_tablet_manager.cc:1398] T 58f576b66ff94cf988d96c2be32232aa P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:19.830528 9344 tablet_bootstrap.cc:492] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:19.830634 9344 ts_tablet_manager.cc:1367] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.014s user 0.002s sys 0.000s
I20240502 01:58:19.831007 9279 tablet_bootstrap.cc:492] T b90c8cce99b94418bf2a4b511cf38e49 P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:19.831110 9279 ts_tablet_manager.cc:1367] T b90c8cce99b94418bf2a4b511cf38e49 P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.000s
I20240502 01:58:19.831117 9344 raft_consensus.cc:348] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.831220 9344 raft_consensus.cc:374] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.831264 9344 raft_consensus.cc:729] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.831383 9344 consensus_queue.cc:260] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.831488 9279 raft_consensus.cc:348] T b90c8cce99b94418bf2a4b511cf38e49 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.831588 9279 raft_consensus.cc:374] T b90c8cce99b94418bf2a4b511cf38e49 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.831626 9279 raft_consensus.cc:729] T b90c8cce99b94418bf2a4b511cf38e49 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.831686 9344 ts_tablet_manager.cc:1398] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:19.831774 9279 consensus_queue.cc:260] T b90c8cce99b94418bf2a4b511cf38e49 P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.831863 9344 tablet_bootstrap.cc:492] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:19.832046 9279 ts_tablet_manager.cc:1398] T b90c8cce99b94418bf2a4b511cf38e49 P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:19.833367 9344 tablet_bootstrap.cc:654] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.834388 9349 tablet_bootstrap.cc:492] T 58f576b66ff94cf988d96c2be32232aa P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:19.834813 9344 tablet_bootstrap.cc:492] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:19.834903 9344 ts_tablet_manager.cc:1367] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:19.835539 9344 raft_consensus.cc:348] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.835650 9344 raft_consensus.cc:374] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.835700 9344 raft_consensus.cc:729] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.835821 9344 consensus_queue.cc:260] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.836064 9344 ts_tablet_manager.cc:1398] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:19.836198 9349 ts_tablet_manager.cc:1367] T 58f576b66ff94cf988d96c2be32232aa P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.000s
I20240502 01:58:19.836544 9349 raft_consensus.cc:348] T 58f576b66ff94cf988d96c2be32232aa P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.836627 9349 raft_consensus.cc:374] T 58f576b66ff94cf988d96c2be32232aa P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.836658 9349 raft_consensus.cc:729] T 58f576b66ff94cf988d96c2be32232aa P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.836778 9349 consensus_queue.cc:260] T 58f576b66ff94cf988d96c2be32232aa P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.837029 9349 ts_tablet_manager.cc:1398] T 58f576b66ff94cf988d96c2be32232aa P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:19.837349 9349 tablet_bootstrap.cc:492] T 10e9d48d98ca4da286c329bf76028c05 P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:19.838155 9349 tablet_bootstrap.cc:654] T 10e9d48d98ca4da286c329bf76028c05 P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:19.839200 9349 tablet_bootstrap.cc:492] T 10e9d48d98ca4da286c329bf76028c05 P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:19.839274 9349 ts_tablet_manager.cc:1367] T 10e9d48d98ca4da286c329bf76028c05 P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 01:58:19.839572 9349 raft_consensus.cc:348] T 10e9d48d98ca4da286c329bf76028c05 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.839728 9349 raft_consensus.cc:374] T 10e9d48d98ca4da286c329bf76028c05 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:19.839777 9349 raft_consensus.cc:729] T 10e9d48d98ca4da286c329bf76028c05 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:19.839896 9349 consensus_queue.cc:260] T 10e9d48d98ca4da286c329bf76028c05 P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.840142 9349 ts_tablet_manager.cc:1398] T 10e9d48d98ca4da286c329bf76028c05 P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:19.874210 9259 raft_consensus.cc:482] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:19.874346 9259 raft_consensus.cc:504] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.874846 9259 leader_election.cc:288] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:19.875236 8372 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "58f576b66ff94cf988d96c2be32232aa" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15" is_pre_election: true
I20240502 01:58:19.875247 8911 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "58f576b66ff94cf988d96c2be32232aa" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" is_pre_election: true
I20240502 01:58:19.875375 8372 raft_consensus.cc:2454] T 58f576b66ff94cf988d96c2be32232aa P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:19.875386 8911 raft_consensus.cc:2454] T 58f576b66ff94cf988d96c2be32232aa P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:19.875680 8574 leader_election.cc:302] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:19.875851 9259 raft_consensus.cc:2781] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:19.875902 9259 raft_consensus.cc:482] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:19.875944 9259 raft_consensus.cc:3037] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.876785 9259 raft_consensus.cc:504] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.877156 9259 leader_election.cc:288] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Requested vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:19.877489 8372 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "58f576b66ff94cf988d96c2be32232aa" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15"
I20240502 01:58:19.877539 8911 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "58f576b66ff94cf988d96c2be32232aa" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
I20240502 01:58:19.877600 8372 raft_consensus.cc:3037] T 58f576b66ff94cf988d96c2be32232aa P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.877647 8911 raft_consensus.cc:3037] T 58f576b66ff94cf988d96c2be32232aa P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.878443 8911 raft_consensus.cc:2454] T 58f576b66ff94cf988d96c2be32232aa P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:19.878487 8372 raft_consensus.cc:2454] T 58f576b66ff94cf988d96c2be32232aa P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:19.878718 8573 leader_election.cc:302] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:19.878986 9259 raft_consensus.cc:2781] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:19.879070 9259 raft_consensus.cc:686] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 LEADER]: Becoming Leader. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Running, Role: LEADER
I20240502 01:58:19.879196 9259 consensus_queue.cc:237] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.880689 8093 catalog_manager.cc:5484] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 reported cstate change: term changed from 0 to 1, leader changed from <none> to ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:19.886678 9343 raft_consensus.cc:482] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:19.886806 9343 raft_consensus.cc:504] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.887179 9343 leader_election.cc:288] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), e325acceaed541869d43c1f762530c2e (127.7.221.67:42267)
I20240502 01:58:19.887558 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b90c8cce99b94418bf2a4b511cf38e49" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" is_pre_election: true
I20240502 01:58:19.887557 8507 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b90c8cce99b94418bf2a4b511cf38e49" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e" is_pre_election: true
I20240502 01:58:19.887678 8240 raft_consensus.cc:2454] T b90c8cce99b94418bf2a4b511cf38e49 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 0.
I20240502 01:58:19.887687 8507 raft_consensus.cc:2454] T b90c8cce99b94418bf2a4b511cf38e49 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 0.
I20240502 01:58:19.887979 8709 leader_election.cc:302] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e325acceaed541869d43c1f762530c2e; no voters:
I20240502 01:58:19.888126 9343 raft_consensus.cc:2781] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:19.888185 9343 raft_consensus.cc:482] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:19.888208 9343 raft_consensus.cc:3037] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.888900 9343 raft_consensus.cc:504] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.889179 9343 leader_election.cc:288] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 election: Requested vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), e325acceaed541869d43c1f762530c2e (127.7.221.67:42267)
I20240502 01:58:19.889497 8507 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b90c8cce99b94418bf2a4b511cf38e49" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e"
I20240502 01:58:19.889503 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b90c8cce99b94418bf2a4b511cf38e49" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
I20240502 01:58:19.889613 8507 raft_consensus.cc:3037] T b90c8cce99b94418bf2a4b511cf38e49 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.889616 8240 raft_consensus.cc:3037] T b90c8cce99b94418bf2a4b511cf38e49 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.890480 8507 raft_consensus.cc:2454] T b90c8cce99b94418bf2a4b511cf38e49 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 1.
I20240502 01:58:19.890522 8240 raft_consensus.cc:2454] T b90c8cce99b94418bf2a4b511cf38e49 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 1.
I20240502 01:58:19.890823 8709 leader_election.cc:302] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e325acceaed541869d43c1f762530c2e; no voters:
I20240502 01:58:19.890990 9343 raft_consensus.cc:2781] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:19.891112 9343 raft_consensus.cc:686] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [term 1 LEADER]: Becoming Leader. State: Replica: 32416365804d40db9d644b5d32301d54, State: Running, Role: LEADER
I20240502 01:58:19.891258 9343 consensus_queue.cc:237] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:19.893033 8093 catalog_manager.cc:5484] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 reported cstate change: term changed from 0 to 1, leader changed from <none> to 32416365804d40db9d644b5d32301d54 (127.7.221.69). New cstate: current_term: 1 leader_uuid: "32416365804d40db9d644b5d32301d54" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: HEALTHY } } }
I20240502 01:58:19.987974 9259 raft_consensus.cc:482] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:19.988104 9259 raft_consensus.cc:504] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.988489 9259 leader_election.cc:288] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:19.988873 8372 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "10e9d48d98ca4da286c329bf76028c05" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15" is_pre_election: true
I20240502 01:58:19.988904 8911 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "10e9d48d98ca4da286c329bf76028c05" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" is_pre_election: true
I20240502 01:58:19.989008 8372 raft_consensus.cc:2454] T 10e9d48d98ca4da286c329bf76028c05 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:19.989050 8911 raft_consensus.cc:2454] T 10e9d48d98ca4da286c329bf76028c05 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:19.989321 8574 leader_election.cc:302] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:19.989483 9259 raft_consensus.cc:2781] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:19.989535 9259 raft_consensus.cc:482] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:19.989559 9259 raft_consensus.cc:3037] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.990314 9259 raft_consensus.cc:504] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.990645 9259 leader_election.cc:288] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Requested vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:19.990947 8372 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "10e9d48d98ca4da286c329bf76028c05" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15"
I20240502 01:58:19.990993 8911 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "10e9d48d98ca4da286c329bf76028c05" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
I20240502 01:58:19.991060 8372 raft_consensus.cc:3037] T 10e9d48d98ca4da286c329bf76028c05 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.991086 8911 raft_consensus.cc:3037] T 10e9d48d98ca4da286c329bf76028c05 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:19.991799 8372 raft_consensus.cc:2454] T 10e9d48d98ca4da286c329bf76028c05 P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:19.991868 8911 raft_consensus.cc:2454] T 10e9d48d98ca4da286c329bf76028c05 P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:19.992097 8574 leader_election.cc:302] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:19.992288 9259 raft_consensus.cc:2781] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:19.992373 9259 raft_consensus.cc:686] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 LEADER]: Becoming Leader. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Running, Role: LEADER
I20240502 01:58:19.992697 9259 consensus_queue.cc:237] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:19.994400 8093 catalog_manager.cc:5484] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 reported cstate change: term changed from 0 to 1, leader changed from <none> to ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:20.124008 9360 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:20.124253 9360 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:20.127193 9360 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_5'
I20240502 01:58:20.215030 8911 raft_consensus.cc:1261] T 10e9d48d98ca4da286c329bf76028c05 P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Refusing update from remote peer ef9a99ad58b84fd2ada7f035f27361b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:20.215744 9000 consensus_queue.cc:1035] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:20.215900 8372 raft_consensus.cc:1261] T 10e9d48d98ca4da286c329bf76028c05 P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Refusing update from remote peer ef9a99ad58b84fd2ada7f035f27361b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:20.217394 8507 raft_consensus.cc:1261] T b90c8cce99b94418bf2a4b511cf38e49 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Refusing update from remote peer 32416365804d40db9d644b5d32301d54: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:20.217396 8240 raft_consensus.cc:1261] T b90c8cce99b94418bf2a4b511cf38e49 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Refusing update from remote peer 32416365804d40db9d644b5d32301d54: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:20.217929 9264 consensus_queue.cc:1035] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:20.218142 9343 consensus_queue.cc:1035] T b90c8cce99b94418bf2a4b511cf38e49 P 32416365804d40db9d644b5d32301d54 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:20.221086 9000 consensus_queue.cc:1035] T 10e9d48d98ca4da286c329bf76028c05 P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:20.230108 8912 raft_consensus.cc:1261] T 58f576b66ff94cf988d96c2be32232aa P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Refusing update from remote peer ef9a99ad58b84fd2ada7f035f27361b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:20.231087 8373 raft_consensus.cc:1261] T 58f576b66ff94cf988d96c2be32232aa P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Refusing update from remote peer ef9a99ad58b84fd2ada7f035f27361b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:20.231575 9259 consensus_queue.cc:1035] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:20.239436 9000 consensus_queue.cc:1035] T 58f576b66ff94cf988d96c2be32232aa P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 122.331 ms
time per row: 0.0611656 ms
I20240502 01:58:20.293524 8094 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34536:
name: "rebalance_test_table_6"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:20.298921 8485 tablet_service.cc:1467] Processing CreateTablet for tablet 3ccb6fa977f849e9af942933c56ef809 (DEFAULT_TABLE table=rebalance_test_table_6 [id=ea7d7557b7f2448c97134c0ae5ba25ea]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.299007 8352 tablet_service.cc:1467] Processing CreateTablet for tablet 206508468d004d0185a73f678beb2427 (DEFAULT_TABLE table=rebalance_test_table_6 [id=ea7d7557b7f2448c97134c0ae5ba25ea]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.299106 8891 tablet_service.cc:1467] Processing CreateTablet for tablet 206508468d004d0185a73f678beb2427 (DEFAULT_TABLE table=rebalance_test_table_6 [id=ea7d7557b7f2448c97134c0ae5ba25ea]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.299244 8352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 206508468d004d0185a73f678beb2427. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.299245 8485 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3ccb6fa977f849e9af942933c56ef809. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.299399 8891 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 206508468d004d0185a73f678beb2427. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.299654 8756 tablet_service.cc:1467] Processing CreateTablet for tablet 3ccb6fa977f849e9af942933c56ef809 (DEFAULT_TABLE table=rebalance_test_table_6 [id=ea7d7557b7f2448c97134c0ae5ba25ea]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.299921 8756 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3ccb6fa977f849e9af942933c56ef809. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.301427 9349 tablet_bootstrap.cc:492] T 206508468d004d0185a73f678beb2427 P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:20.301457 8220 tablet_service.cc:1467] Processing CreateTablet for tablet 3ccb6fa977f849e9af942933c56ef809 (DEFAULT_TABLE table=rebalance_test_table_6 [id=ea7d7557b7f2448c97134c0ae5ba25ea]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.301690 8220 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3ccb6fa977f849e9af942933c56ef809. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.302034 9214 tablet_bootstrap.cc:492] T 3ccb6fa977f849e9af942933c56ef809 P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:20.302115 9346 tablet_bootstrap.cc:492] T 206508468d004d0185a73f678beb2427 P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:20.302443 9349 tablet_bootstrap.cc:654] T 206508468d004d0185a73f678beb2427 P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.302749 9279 tablet_bootstrap.cc:492] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:20.302860 9214 tablet_bootstrap.cc:654] T 3ccb6fa977f849e9af942933c56ef809 P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.303118 9346 tablet_bootstrap.cc:654] T 206508468d004d0185a73f678beb2427 P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.303550 9349 tablet_bootstrap.cc:492] T 206508468d004d0185a73f678beb2427 P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:20.303643 9349 ts_tablet_manager.cc:1367] T 206508468d004d0185a73f678beb2427 P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.001s
I20240502 01:58:20.303663 9279 tablet_bootstrap.cc:654] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.304041 9349 raft_consensus.cc:348] T 206508468d004d0185a73f678beb2427 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.304144 9349 raft_consensus.cc:374] T 206508468d004d0185a73f678beb2427 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.304170 9349 raft_consensus.cc:729] T 206508468d004d0185a73f678beb2427 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.304280 9349 consensus_queue.cc:260] T 206508468d004d0185a73f678beb2427 P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.304544 9349 ts_tablet_manager.cc:1398] T 206508468d004d0185a73f678beb2427 P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.001s user 0.001s sys 0.001s
I20240502 01:58:20.305222 8219 tablet_service.cc:1467] Processing CreateTablet for tablet 92f03a0caead4668969af4eefe0b6be3 (DEFAULT_TABLE table=rebalance_test_table_6 [id=ea7d7557b7f2448c97134c0ae5ba25ea]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.305501 8219 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 92f03a0caead4668969af4eefe0b6be3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.306664 9272 tablet_bootstrap.cc:492] T 3ccb6fa977f849e9af942933c56ef809 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:20.307260 9279 tablet_bootstrap.cc:492] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:20.307339 9279 ts_tablet_manager.cc:1367] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 01:58:20.307384 9272 tablet_bootstrap.cc:654] T 3ccb6fa977f849e9af942933c56ef809 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.307761 9279 raft_consensus.cc:348] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:20.307886 9279 raft_consensus.cc:374] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.307924 9279 raft_consensus.cc:729] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.308064 9279 consensus_queue.cc:260] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:20.308316 9279 ts_tablet_manager.cc:1398] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:20.308358 9272 tablet_bootstrap.cc:492] T 3ccb6fa977f849e9af942933c56ef809 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:20.308423 9272 ts_tablet_manager.cc:1367] T 3ccb6fa977f849e9af942933c56ef809 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 01:58:20.308761 9272 raft_consensus.cc:348] T 3ccb6fa977f849e9af942933c56ef809 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:20.308856 9272 raft_consensus.cc:374] T 3ccb6fa977f849e9af942933c56ef809 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.308879 9272 raft_consensus.cc:729] T 3ccb6fa977f849e9af942933c56ef809 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.309015 9272 consensus_queue.cc:260] T 3ccb6fa977f849e9af942933c56ef809 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:20.309252 9272 ts_tablet_manager.cc:1398] T 3ccb6fa977f849e9af942933c56ef809 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:20.309389 9272 tablet_bootstrap.cc:492] T 92f03a0caead4668969af4eefe0b6be3 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:20.309834 8485 tablet_service.cc:1467] Processing CreateTablet for tablet 92f03a0caead4668969af4eefe0b6be3 (DEFAULT_TABLE table=rebalance_test_table_6 [id=ea7d7557b7f2448c97134c0ae5ba25ea]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.310078 8485 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 92f03a0caead4668969af4eefe0b6be3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.310154 9272 tablet_bootstrap.cc:654] T 92f03a0caead4668969af4eefe0b6be3 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.312999 8621 tablet_service.cc:1467] Processing CreateTablet for tablet 206508468d004d0185a73f678beb2427 (DEFAULT_TABLE table=rebalance_test_table_6 [id=ea7d7557b7f2448c97134c0ae5ba25ea]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.313274 8621 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 206508468d004d0185a73f678beb2427. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.313761 9272 tablet_bootstrap.cc:492] T 92f03a0caead4668969af4eefe0b6be3 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:20.313843 9272 ts_tablet_manager.cc:1367] T 92f03a0caead4668969af4eefe0b6be3 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.001s
I20240502 01:58:20.314253 9272 raft_consensus.cc:348] T 92f03a0caead4668969af4eefe0b6be3 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.314328 9346 tablet_bootstrap.cc:492] T 206508468d004d0185a73f678beb2427 P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:20.314354 9272 raft_consensus.cc:374] T 92f03a0caead4668969af4eefe0b6be3 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.314391 9272 raft_consensus.cc:729] T 92f03a0caead4668969af4eefe0b6be3 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.314415 9346 ts_tablet_manager.cc:1367] T 206508468d004d0185a73f678beb2427 P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.012s user 0.002s sys 0.000s
I20240502 01:58:20.314523 9272 consensus_queue.cc:260] T 92f03a0caead4668969af4eefe0b6be3 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.314801 9272 ts_tablet_manager.cc:1398] T 92f03a0caead4668969af4eefe0b6be3 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:20.314924 9346 raft_consensus.cc:348] T 206508468d004d0185a73f678beb2427 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.315073 9346 raft_consensus.cc:374] T 206508468d004d0185a73f678beb2427 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.315112 9346 raft_consensus.cc:729] T 206508468d004d0185a73f678beb2427 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.315232 9279 tablet_bootstrap.cc:492] T 92f03a0caead4668969af4eefe0b6be3 P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:20.315238 9346 consensus_queue.cc:260] T 206508468d004d0185a73f678beb2427 P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.315508 9346 ts_tablet_manager.cc:1398] T 206508468d004d0185a73f678beb2427 P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:20.315654 9344 tablet_bootstrap.cc:492] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:20.316468 9344 tablet_bootstrap.cc:654] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.316699 9279 tablet_bootstrap.cc:654] T 92f03a0caead4668969af4eefe0b6be3 P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.317909 8756 tablet_service.cc:1467] Processing CreateTablet for tablet 92f03a0caead4668969af4eefe0b6be3 (DEFAULT_TABLE table=rebalance_test_table_6 [id=ea7d7557b7f2448c97134c0ae5ba25ea]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.318115 9279 tablet_bootstrap.cc:492] T 92f03a0caead4668969af4eefe0b6be3 P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:20.318183 8756 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 92f03a0caead4668969af4eefe0b6be3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.318200 9279 ts_tablet_manager.cc:1367] T 92f03a0caead4668969af4eefe0b6be3 P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:20.318538 9279 raft_consensus.cc:348] T 92f03a0caead4668969af4eefe0b6be3 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.318696 9279 raft_consensus.cc:374] T 92f03a0caead4668969af4eefe0b6be3 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.318732 9279 raft_consensus.cc:729] T 92f03a0caead4668969af4eefe0b6be3 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.318856 9279 consensus_queue.cc:260] T 92f03a0caead4668969af4eefe0b6be3 P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.319144 9279 ts_tablet_manager.cc:1398] T 92f03a0caead4668969af4eefe0b6be3 P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:20.319538 9214 tablet_bootstrap.cc:492] T 3ccb6fa977f849e9af942933c56ef809 P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:20.319613 9214 ts_tablet_manager.cc:1367] T 3ccb6fa977f849e9af942933c56ef809 P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.018s user 0.002s sys 0.000s
I20240502 01:58:20.320067 9214 raft_consensus.cc:348] T 3ccb6fa977f849e9af942933c56ef809 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:20.320174 9214 raft_consensus.cc:374] T 3ccb6fa977f849e9af942933c56ef809 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.320210 9214 raft_consensus.cc:729] T 3ccb6fa977f849e9af942933c56ef809 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.320317 9214 consensus_queue.cc:260] T 3ccb6fa977f849e9af942933c56ef809 P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:20.320578 9214 ts_tablet_manager.cc:1398] T 3ccb6fa977f849e9af942933c56ef809 P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:20.320917 9214 tablet_bootstrap.cc:492] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:20.321074 9344 tablet_bootstrap.cc:492] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:20.321161 9344 ts_tablet_manager.cc:1367] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.001s
I20240502 01:58:20.321553 9344 raft_consensus.cc:348] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.321651 9214 tablet_bootstrap.cc:654] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.321641 9344 raft_consensus.cc:374] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.321677 9344 raft_consensus.cc:729] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.321825 9344 consensus_queue.cc:260] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.322065 9344 ts_tablet_manager.cc:1398] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:20.323252 9214 tablet_bootstrap.cc:492] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:20.323354 9214 ts_tablet_manager.cc:1367] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:20.324390 9214 raft_consensus.cc:348] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.324540 9214 raft_consensus.cc:374] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.324591 9214 raft_consensus.cc:729] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.324734 9214 consensus_queue.cc:260] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.325017 9214 ts_tablet_manager.cc:1398] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:20.335383 9259 raft_consensus.cc:482] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:20.335495 9259 raft_consensus.cc:504] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.335893 9259 leader_election.cc:288] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:20.336223 8372 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "206508468d004d0185a73f678beb2427" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15" is_pre_election: true
I20240502 01:58:20.336288 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "206508468d004d0185a73f678beb2427" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" is_pre_election: true
I20240502 01:58:20.336370 8372 raft_consensus.cc:2454] T 206508468d004d0185a73f678beb2427 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:20.336437 8912 raft_consensus.cc:2454] T 206508468d004d0185a73f678beb2427 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:20.336747 8574 leader_election.cc:302] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:20.336925 9259 raft_consensus.cc:2781] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:20.336967 9259 raft_consensus.cc:482] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:20.337013 9259 raft_consensus.cc:3037] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.337760 9259 raft_consensus.cc:504] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.338235 9259 leader_election.cc:288] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Requested vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:20.338375 8372 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "206508468d004d0185a73f678beb2427" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15"
I20240502 01:58:20.338493 8372 raft_consensus.cc:3037] T 206508468d004d0185a73f678beb2427 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.338680 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "206508468d004d0185a73f678beb2427" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
I20240502 01:58:20.338819 8912 raft_consensus.cc:3037] T 206508468d004d0185a73f678beb2427 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.339522 8372 raft_consensus.cc:2454] T 206508468d004d0185a73f678beb2427 P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:20.339794 8912 raft_consensus.cc:2454] T 206508468d004d0185a73f678beb2427 P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:20.339799 8574 leader_election.cc:302] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:20.339949 9259 raft_consensus.cc:2781] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:20.340035 9259 raft_consensus.cc:686] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 LEADER]: Becoming Leader. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Running, Role: LEADER
I20240502 01:58:20.340173 9259 consensus_queue.cc:237] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.341995 8094 catalog_manager.cc:5484] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 reported cstate change: term changed from 0 to 1, leader changed from <none> to ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: HEALTHY } } }
I20240502 01:58:20.428249 9343 raft_consensus.cc:482] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:20.428395 9343 raft_consensus.cc:504] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.428856 9343 leader_election.cc:288] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), e325acceaed541869d43c1f762530c2e (127.7.221.67:42267)
I20240502 01:58:20.429350 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "92f03a0caead4668969af4eefe0b6be3" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" is_pre_election: true
I20240502 01:58:20.429383 8506 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "92f03a0caead4668969af4eefe0b6be3" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e" is_pre_election: true
I20240502 01:58:20.429497 8240 raft_consensus.cc:2454] T 92f03a0caead4668969af4eefe0b6be3 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 0.
I20240502 01:58:20.429526 8506 raft_consensus.cc:2454] T 92f03a0caead4668969af4eefe0b6be3 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 0.
I20240502 01:58:20.429844 8710 leader_election.cc:302] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:20.430014 9343 raft_consensus.cc:2781] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:20.430088 9343 raft_consensus.cc:482] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:20.430114 9343 raft_consensus.cc:3037] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.431008 9343 raft_consensus.cc:504] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.431365 9343 leader_election.cc:288] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 election: Requested vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), e325acceaed541869d43c1f762530c2e (127.7.221.67:42267)
I20240502 01:58:20.431685 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "92f03a0caead4668969af4eefe0b6be3" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
I20240502 01:58:20.431710 8506 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "92f03a0caead4668969af4eefe0b6be3" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e"
I20240502 01:58:20.431807 8240 raft_consensus.cc:3037] T 92f03a0caead4668969af4eefe0b6be3 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.431825 8506 raft_consensus.cc:3037] T 92f03a0caead4668969af4eefe0b6be3 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.432575 8506 raft_consensus.cc:2454] T 92f03a0caead4668969af4eefe0b6be3 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 1.
I20240502 01:58:20.432796 8240 raft_consensus.cc:2454] T 92f03a0caead4668969af4eefe0b6be3 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 1.
I20240502 01:58:20.432832 8709 leader_election.cc:302] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e325acceaed541869d43c1f762530c2e; no voters:
I20240502 01:58:20.432991 9343 raft_consensus.cc:2781] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:20.433202 9343 raft_consensus.cc:686] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [term 1 LEADER]: Becoming Leader. State: Replica: 32416365804d40db9d644b5d32301d54, State: Running, Role: LEADER
I20240502 01:58:20.433369 9343 consensus_queue.cc:237] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.435139 8094 catalog_manager.cc:5484] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 reported cstate change: term changed from 0 to 1, leader changed from <none> to 32416365804d40db9d644b5d32301d54 (127.7.221.69). New cstate: current_term: 1 leader_uuid: "32416365804d40db9d644b5d32301d54" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:20.501556 9267 raft_consensus.cc:482] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:20.501693 9267 raft_consensus.cc:504] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:20.502156 9267 leader_election.cc:288] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775)
I20240502 01:58:20.502529 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3ccb6fa977f849e9af942933c56ef809" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" is_pre_election: true
I20240502 01:58:20.502539 8776 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3ccb6fa977f849e9af942933c56ef809" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54" is_pre_election: true
I20240502 01:58:20.502686 8776 raft_consensus.cc:2454] T 3ccb6fa977f849e9af942933c56ef809 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 0.
I20240502 01:58:20.502702 8240 raft_consensus.cc:2454] T 3ccb6fa977f849e9af942933c56ef809 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 0.
I20240502 01:58:20.503005 8439 leader_election.cc:302] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e325acceaed541869d43c1f762530c2e; no voters:
I20240502 01:58:20.503209 9267 raft_consensus.cc:2781] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:20.503297 9267 raft_consensus.cc:482] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:20.503331 9267 raft_consensus.cc:3037] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.504397 9267 raft_consensus.cc:504] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:20.504735 9267 leader_election.cc:288] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 election: Requested vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775)
I20240502 01:58:20.505062 8776 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3ccb6fa977f849e9af942933c56ef809" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54"
I20240502 01:58:20.505173 8776 raft_consensus.cc:3037] T 3ccb6fa977f849e9af942933c56ef809 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.506009 8776 raft_consensus.cc:2454] T 3ccb6fa977f849e9af942933c56ef809 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 1.
I20240502 01:58:20.506281 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3ccb6fa977f849e9af942933c56ef809" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
I20240502 01:58:20.506312 8439 leader_election.cc:302] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e325acceaed541869d43c1f762530c2e; no voters:
I20240502 01:58:20.506397 8240 raft_consensus.cc:3037] T 3ccb6fa977f849e9af942933c56ef809 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.506485 9267 raft_consensus.cc:2781] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:20.506644 9267 raft_consensus.cc:686] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [term 1 LEADER]: Becoming Leader. State: Replica: e325acceaed541869d43c1f762530c2e, State: Running, Role: LEADER
I20240502 01:58:20.506798 9267 consensus_queue.cc:237] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:20.507369 8240 raft_consensus.cc:2454] T 3ccb6fa977f849e9af942933c56ef809 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 1.
I20240502 01:58:20.508818 8094 catalog_manager.cc:5484] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e reported cstate change: term changed from 0 to 1, leader changed from <none> to e325acceaed541869d43c1f762530c2e (127.7.221.67). New cstate: current_term: 1 leader_uuid: "e325acceaed541869d43c1f762530c2e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:20.655217 9414 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:20.655449 9414 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:20.658552 9414 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_6'
I20240502 01:58:20.718238 9000 consensus_queue.cc:1035] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:20.721406 9161 consensus_queue.cc:1035] T 206508468d004d0185a73f678beb2427 P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:20.751389 8240 raft_consensus.cc:1261] T 3ccb6fa977f849e9af942933c56ef809 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Refusing update from remote peer e325acceaed541869d43c1f762530c2e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:20.751659 8776 raft_consensus.cc:1261] T 3ccb6fa977f849e9af942933c56ef809 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Refusing update from remote peer e325acceaed541869d43c1f762530c2e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:20.752377 9267 consensus_queue.cc:1035] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [LEADER]: Connected to new peer: Peer: permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:20.759781 9267 consensus_queue.cc:1035] T 3ccb6fa977f849e9af942933c56ef809 P e325acceaed541869d43c1f762530c2e [LEADER]: Connected to new peer: Peer: permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:20.771452 8506 raft_consensus.cc:1261] T 92f03a0caead4668969af4eefe0b6be3 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Refusing update from remote peer 32416365804d40db9d644b5d32301d54: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:20.772161 9264 consensus_queue.cc:1035] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:20.774844 8240 raft_consensus.cc:1261] T 92f03a0caead4668969af4eefe0b6be3 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Refusing update from remote peer 32416365804d40db9d644b5d32301d54: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:20.775549 9264 consensus_queue.cc:1035] T 92f03a0caead4668969af4eefe0b6be3 P 32416365804d40db9d644b5d32301d54 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 113.245 ms
time per row: 0.0566222 ms
I20240502 01:58:20.815549 8094 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34536:
name: "rebalance_test_table_7"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:20.820968 8891 tablet_service.cc:1467] Processing CreateTablet for tablet fc088bb34c27498185a161f2bae4ab3c (DEFAULT_TABLE table=rebalance_test_table_7 [id=cb68ac68e0fb4e89a83d1310b8792107]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.821020 8352 tablet_service.cc:1467] Processing CreateTablet for tablet fc088bb34c27498185a161f2bae4ab3c (DEFAULT_TABLE table=rebalance_test_table_7 [id=cb68ac68e0fb4e89a83d1310b8792107]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.821346 8352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc088bb34c27498185a161f2bae4ab3c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.821349 8891 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc088bb34c27498185a161f2bae4ab3c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.821622 8621 tablet_service.cc:1467] Processing CreateTablet for tablet fc088bb34c27498185a161f2bae4ab3c (DEFAULT_TABLE table=rebalance_test_table_7 [id=cb68ac68e0fb4e89a83d1310b8792107]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.821830 8220 tablet_service.cc:1467] Processing CreateTablet for tablet 209389679cb5447291f54dfe286c67ea (DEFAULT_TABLE table=rebalance_test_table_7 [id=cb68ac68e0fb4e89a83d1310b8792107]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.821918 8621 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc088bb34c27498185a161f2bae4ab3c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.822113 8220 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 209389679cb5447291f54dfe286c67ea. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.824295 9344 tablet_bootstrap.cc:492] T fc088bb34c27498185a161f2bae4ab3c P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:20.824317 9473 tablet_bootstrap.cc:492] T 209389679cb5447291f54dfe286c67ea P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:20.824343 8620 tablet_service.cc:1467] Processing CreateTablet for tablet b1d2473ff348402999cea0f566f31d7f (DEFAULT_TABLE table=rebalance_test_table_7 [id=cb68ac68e0fb4e89a83d1310b8792107]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.824620 8620 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b1d2473ff348402999cea0f566f31d7f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.825016 8485 tablet_service.cc:1467] Processing CreateTablet for tablet 209389679cb5447291f54dfe286c67ea (DEFAULT_TABLE table=rebalance_test_table_7 [id=cb68ac68e0fb4e89a83d1310b8792107]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.825084 8352 tablet_service.cc:1467] Processing CreateTablet for tablet b1d2473ff348402999cea0f566f31d7f (DEFAULT_TABLE table=rebalance_test_table_7 [id=cb68ac68e0fb4e89a83d1310b8792107]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.825213 9344 tablet_bootstrap.cc:654] T fc088bb34c27498185a161f2bae4ab3c P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.825306 8485 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 209389679cb5447291f54dfe286c67ea. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.825316 8352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b1d2473ff348402999cea0f566f31d7f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.827332 8891 tablet_service.cc:1467] Processing CreateTablet for tablet b1d2473ff348402999cea0f566f31d7f (DEFAULT_TABLE table=rebalance_test_table_7 [id=cb68ac68e0fb4e89a83d1310b8792107]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.827428 9473 tablet_bootstrap.cc:654] T 209389679cb5447291f54dfe286c67ea P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.827584 8891 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b1d2473ff348402999cea0f566f31d7f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.828431 9473 tablet_bootstrap.cc:492] T 209389679cb5447291f54dfe286c67ea P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:20.828508 9473 ts_tablet_manager.cc:1367] T 209389679cb5447291f54dfe286c67ea P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 01:58:20.828589 9472 tablet_bootstrap.cc:492] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:20.829124 9473 raft_consensus.cc:348] T 209389679cb5447291f54dfe286c67ea P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.829229 9473 raft_consensus.cc:374] T 209389679cb5447291f54dfe286c67ea P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.829259 9473 raft_consensus.cc:729] T 209389679cb5447291f54dfe286c67ea P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.829396 9473 consensus_queue.cc:260] T 209389679cb5447291f54dfe286c67ea P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.829988 8756 tablet_service.cc:1467] Processing CreateTablet for tablet 209389679cb5447291f54dfe286c67ea (DEFAULT_TABLE table=rebalance_test_table_7 [id=cb68ac68e0fb4e89a83d1310b8792107]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:20.830256 9472 tablet_bootstrap.cc:654] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.830267 9471 tablet_bootstrap.cc:492] T fc088bb34c27498185a161f2bae4ab3c P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:20.830332 8756 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 209389679cb5447291f54dfe286c67ea. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:20.830539 9344 tablet_bootstrap.cc:492] T fc088bb34c27498185a161f2bae4ab3c P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:20.830650 9344 ts_tablet_manager.cc:1367] T fc088bb34c27498185a161f2bae4ab3c P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.001s
I20240502 01:58:20.831056 9344 raft_consensus.cc:348] T fc088bb34c27498185a161f2bae4ab3c P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.831148 9471 tablet_bootstrap.cc:654] T fc088bb34c27498185a161f2bae4ab3c P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.831166 9344 raft_consensus.cc:374] T fc088bb34c27498185a161f2bae4ab3c P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.831193 9344 raft_consensus.cc:729] T fc088bb34c27498185a161f2bae4ab3c P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.831300 9344 consensus_queue.cc:260] T fc088bb34c27498185a161f2bae4ab3c P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.831554 9344 ts_tablet_manager.cc:1398] T fc088bb34c27498185a161f2bae4ab3c P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:20.831712 9344 tablet_bootstrap.cc:492] T b1d2473ff348402999cea0f566f31d7f P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:20.832077 9471 tablet_bootstrap.cc:492] T fc088bb34c27498185a161f2bae4ab3c P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:20.832149 9471 ts_tablet_manager.cc:1367] T fc088bb34c27498185a161f2bae4ab3c P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:20.832473 9471 raft_consensus.cc:348] T fc088bb34c27498185a161f2bae4ab3c P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.832545 9471 raft_consensus.cc:374] T fc088bb34c27498185a161f2bae4ab3c P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.832566 9471 raft_consensus.cc:729] T fc088bb34c27498185a161f2bae4ab3c P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.832640 9344 tablet_bootstrap.cc:654] T b1d2473ff348402999cea0f566f31d7f P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.832682 9471 consensus_queue.cc:260] T fc088bb34c27498185a161f2bae4ab3c P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.832846 9479 tablet_bootstrap.cc:492] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:20.833480 9471 ts_tablet_manager.cc:1398] T fc088bb34c27498185a161f2bae4ab3c P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:20.833628 9471 tablet_bootstrap.cc:492] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:20.833750 9479 tablet_bootstrap.cc:654] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.834219 9344 tablet_bootstrap.cc:492] T b1d2473ff348402999cea0f566f31d7f P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:20.834295 9344 ts_tablet_manager.cc:1367] T b1d2473ff348402999cea0f566f31d7f P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:20.834443 9471 tablet_bootstrap.cc:654] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.831724 9475 tablet_bootstrap.cc:492] T 209389679cb5447291f54dfe286c67ea P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:20.834792 9344 raft_consensus.cc:348] T b1d2473ff348402999cea0f566f31d7f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:20.834935 9344 raft_consensus.cc:374] T b1d2473ff348402999cea0f566f31d7f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.834975 9344 raft_consensus.cc:729] T b1d2473ff348402999cea0f566f31d7f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.835088 9344 consensus_queue.cc:260] T b1d2473ff348402999cea0f566f31d7f P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:20.835371 9344 ts_tablet_manager.cc:1398] T b1d2473ff348402999cea0f566f31d7f P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:20.835790 9475 tablet_bootstrap.cc:654] T 209389679cb5447291f54dfe286c67ea P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.836637 9471 tablet_bootstrap.cc:492] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:20.836720 9471 ts_tablet_manager.cc:1367] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:20.837117 9471 raft_consensus.cc:348] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:20.837198 9471 raft_consensus.cc:374] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.837230 9471 raft_consensus.cc:729] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.837373 9471 consensus_queue.cc:260] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:20.837641 9471 ts_tablet_manager.cc:1398] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:20.838124 9473 ts_tablet_manager.cc:1398] T 209389679cb5447291f54dfe286c67ea P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.010s user 0.001s sys 0.000s
I20240502 01:58:20.838155 9475 tablet_bootstrap.cc:492] T 209389679cb5447291f54dfe286c67ea P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:20.838238 9475 ts_tablet_manager.cc:1367] T 209389679cb5447291f54dfe286c67ea P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 01:58:20.838734 9475 raft_consensus.cc:348] T 209389679cb5447291f54dfe286c67ea P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.838910 9475 raft_consensus.cc:374] T 209389679cb5447291f54dfe286c67ea P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.838995 9475 raft_consensus.cc:729] T 209389679cb5447291f54dfe286c67ea P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.839201 9479 tablet_bootstrap.cc:492] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:20.839222 9472 tablet_bootstrap.cc:492] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:20.839300 9472 ts_tablet_manager.cc:1367] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.000s
I20240502 01:58:20.839294 9479 ts_tablet_manager.cc:1367] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240502 01:58:20.839236 9475 consensus_queue.cc:260] T 209389679cb5447291f54dfe286c67ea P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.839691 9472 raft_consensus.cc:348] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.839689 9479 raft_consensus.cc:348] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.839788 9472 raft_consensus.cc:374] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.839797 9479 raft_consensus.cc:374] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.839825 9472 raft_consensus.cc:729] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.839829 9479 raft_consensus.cc:729] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.839946 9479 consensus_queue.cc:260] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.839946 9472 consensus_queue.cc:260] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.840195 9479 ts_tablet_manager.cc:1398] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:20.840200 9472 ts_tablet_manager.cc:1398] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:20.840360 9472 tablet_bootstrap.cc:492] T b1d2473ff348402999cea0f566f31d7f P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:20.840492 9475 ts_tablet_manager.cc:1398] T 209389679cb5447291f54dfe286c67ea P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:20.841218 9472 tablet_bootstrap.cc:654] T b1d2473ff348402999cea0f566f31d7f P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:20.842828 9472 tablet_bootstrap.cc:492] T b1d2473ff348402999cea0f566f31d7f P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:20.842911 9472 ts_tablet_manager.cc:1367] T b1d2473ff348402999cea0f566f31d7f P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:20.843290 9472 raft_consensus.cc:348] T b1d2473ff348402999cea0f566f31d7f P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:20.843394 9472 raft_consensus.cc:374] T b1d2473ff348402999cea0f566f31d7f P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:20.843425 9472 raft_consensus.cc:729] T b1d2473ff348402999cea0f566f31d7f P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:20.843624 9472 consensus_queue.cc:260] T b1d2473ff348402999cea0f566f31d7f P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:20.843933 9472 ts_tablet_manager.cc:1398] T b1d2473ff348402999cea0f566f31d7f P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:20.901228 9343 raft_consensus.cc:482] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:20.901389 9343 raft_consensus.cc:504] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.901870 9343 leader_election.cc:288] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), e325acceaed541869d43c1f762530c2e (127.7.221.67:42267)
I20240502 01:58:20.902268 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "209389679cb5447291f54dfe286c67ea" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" is_pre_election: true
I20240502 01:58:20.902318 8506 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "209389679cb5447291f54dfe286c67ea" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e" is_pre_election: true
I20240502 01:58:20.902426 8240 raft_consensus.cc:2454] T 209389679cb5447291f54dfe286c67ea P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 0.
I20240502 01:58:20.902487 8506 raft_consensus.cc:2454] T 209389679cb5447291f54dfe286c67ea P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 0.
I20240502 01:58:20.902726 8710 leader_election.cc:302] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:20.902902 9343 raft_consensus.cc:2781] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:20.902951 9343 raft_consensus.cc:482] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:20.903010 9343 raft_consensus.cc:3037] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.903805 9343 raft_consensus.cc:504] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.904109 9343 leader_election.cc:288] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 election: Requested vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), e325acceaed541869d43c1f762530c2e (127.7.221.67:42267)
I20240502 01:58:20.904486 8506 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "209389679cb5447291f54dfe286c67ea" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e"
I20240502 01:58:20.904486 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "209389679cb5447291f54dfe286c67ea" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
I20240502 01:58:20.904587 8506 raft_consensus.cc:3037] T 209389679cb5447291f54dfe286c67ea P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.904623 8240 raft_consensus.cc:3037] T 209389679cb5447291f54dfe286c67ea P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.905321 8506 raft_consensus.cc:2454] T 209389679cb5447291f54dfe286c67ea P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 1.
I20240502 01:58:20.905440 8240 raft_consensus.cc:2454] T 209389679cb5447291f54dfe286c67ea P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 1.
I20240502 01:58:20.905589 8709 leader_election.cc:302] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e325acceaed541869d43c1f762530c2e; no voters:
I20240502 01:58:20.905788 9343 raft_consensus.cc:2781] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:20.905856 9343 raft_consensus.cc:686] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [term 1 LEADER]: Becoming Leader. State: Replica: 32416365804d40db9d644b5d32301d54, State: Running, Role: LEADER
I20240502 01:58:20.905997 9343 consensus_queue.cc:237] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:20.907853 8092 catalog_manager.cc:5484] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 reported cstate change: term changed from 0 to 1, leader changed from <none> to 32416365804d40db9d644b5d32301d54 (127.7.221.69). New cstate: current_term: 1 leader_uuid: "32416365804d40db9d644b5d32301d54" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:20.931908 9325 raft_consensus.cc:482] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:20.932067 9325 raft_consensus.cc:504] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:20.932547 9325 leader_election.cc:288] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:20.932992 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b1d2473ff348402999cea0f566f31d7f" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" is_pre_election: true
I20240502 01:58:20.933171 8642 raft_consensus.cc:2454] T b1d2473ff348402999cea0f566f31d7f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 73ee70f488334b8f8ffb83aef6456e15 in term 0.
I20240502 01:58:20.933540 8305 leader_election.cc:302] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:20.933705 9325 raft_consensus.cc:2781] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:20.933753 9325 raft_consensus.cc:482] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:20.933789 9325 raft_consensus.cc:3037] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.934885 9325 raft_consensus.cc:504] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:20.935250 9325 leader_election.cc:288] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 election: Requested vote from peers ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:20.935665 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b1d2473ff348402999cea0f566f31d7f" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1"
I20240502 01:58:20.935784 8642 raft_consensus.cc:3037] T b1d2473ff348402999cea0f566f31d7f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.936709 8642 raft_consensus.cc:2454] T b1d2473ff348402999cea0f566f31d7f P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 73ee70f488334b8f8ffb83aef6456e15 in term 1.
I20240502 01:58:20.937157 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b1d2473ff348402999cea0f566f31d7f" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" is_pre_election: true
I20240502 01:58:20.937220 8305 leader_election.cc:302] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:20.937244 8911 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b1d2473ff348402999cea0f566f31d7f" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
I20240502 01:58:20.937326 8912 raft_consensus.cc:2454] T b1d2473ff348402999cea0f566f31d7f P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 73ee70f488334b8f8ffb83aef6456e15 in term 0.
I20240502 01:58:20.937489 9325 raft_consensus.cc:2781] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:20.937703 9325 raft_consensus.cc:686] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [term 1 LEADER]: Becoming Leader. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Running, Role: LEADER
I20240502 01:58:20.937852 9325 consensus_queue.cc:237] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:20.939715 8092 catalog_manager.cc:5484] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 reported cstate change: term changed from 0 to 1, leader changed from <none> to 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "73ee70f488334b8f8ffb83aef6456e15" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:20.950093 9134 raft_consensus.cc:482] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:20.950212 9134 raft_consensus.cc:504] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.950616 9134 leader_election.cc:288] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021)
I20240502 01:58:20.950985 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc088bb34c27498185a161f2bae4ab3c" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15" is_pre_election: true
I20240502 01:58:20.951001 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc088bb34c27498185a161f2bae4ab3c" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" is_pre_election: true
I20240502 01:58:20.951123 8642 raft_consensus.cc:2454] T fc088bb34c27498185a161f2bae4ab3c P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 0.
I20240502 01:58:20.951126 8373 raft_consensus.cc:2454] T fc088bb34c27498185a161f2bae4ab3c P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 0.
I20240502 01:58:20.951409 8844 leader_election.cc:302] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:20.951578 9134 raft_consensus.cc:2781] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:20.951638 9134 raft_consensus.cc:482] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:20.951668 9134 raft_consensus.cc:3037] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.952502 9134 raft_consensus.cc:504] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.952834 9134 leader_election.cc:288] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 election: Requested vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021)
I20240502 01:58:20.953186 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc088bb34c27498185a161f2bae4ab3c" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1"
I20240502 01:58:20.953341 8642 raft_consensus.cc:3037] T fc088bb34c27498185a161f2bae4ab3c P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.953346 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc088bb34c27498185a161f2bae4ab3c" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15"
I20240502 01:58:20.953462 8373 raft_consensus.cc:3037] T fc088bb34c27498185a161f2bae4ab3c P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:20.954298 8642 raft_consensus.cc:2454] T fc088bb34c27498185a161f2bae4ab3c P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 1.
I20240502 01:58:20.954337 8373 raft_consensus.cc:2454] T fc088bb34c27498185a161f2bae4ab3c P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 1.
I20240502 01:58:20.954581 8844 leader_election.cc:302] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:20.954782 9134 raft_consensus.cc:2781] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:20.954878 9134 raft_consensus.cc:686] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 LEADER]: Becoming Leader. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Running, Role: LEADER
I20240502 01:58:20.955013 9134 consensus_queue.cc:237] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:20.956581 8092 catalog_manager.cc:5484] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb reported cstate change: term changed from 0 to 1, leader changed from <none> to 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70). New cstate: current_term: 1 leader_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:21.092187 9491 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:21.092425 9491 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:21.095417 9491 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_7'
I20240502 01:58:21.178308 8642 raft_consensus.cc:1261] T fc088bb34c27498185a161f2bae4ab3c P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Refusing update from remote peer 1caeef741efa452fa9bb1fd3d47e65eb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:21.179324 8373 raft_consensus.cc:1261] T fc088bb34c27498185a161f2bae4ab3c P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Refusing update from remote peer 1caeef741efa452fa9bb1fd3d47e65eb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:21.179435 9394 consensus_queue.cc:1035] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [LEADER]: Connected to new peer: Peer: permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:21.179723 9134 consensus_queue.cc:1035] T fc088bb34c27498185a161f2bae4ab3c P 1caeef741efa452fa9bb1fd3d47e65eb [LEADER]: Connected to new peer: Peer: permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:21.184492 8912 raft_consensus.cc:3037] T b1d2473ff348402999cea0f566f31d7f P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.185635 8912 raft_consensus.cc:1261] T b1d2473ff348402999cea0f566f31d7f P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Refusing update from remote peer 73ee70f488334b8f8ffb83aef6456e15: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:21.186255 9488 consensus_queue.cc:1035] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:21.186870 8641 raft_consensus.cc:1261] T b1d2473ff348402999cea0f566f31d7f P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Refusing update from remote peer 73ee70f488334b8f8ffb83aef6456e15: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:21.186870 8506 raft_consensus.cc:1261] T 209389679cb5447291f54dfe286c67ea P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Refusing update from remote peer 32416365804d40db9d644b5d32301d54: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:21.187316 9343 consensus_queue.cc:1035] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:21.188174 9488 consensus_queue.cc:1035] T b1d2473ff348402999cea0f566f31d7f P 73ee70f488334b8f8ffb83aef6456e15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:21.193763 8240 raft_consensus.cc:1261] T 209389679cb5447291f54dfe286c67ea P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Refusing update from remote peer 32416365804d40db9d644b5d32301d54: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:21.194855 9343 consensus_queue.cc:1035] T 209389679cb5447291f54dfe286c67ea P 32416365804d40db9d644b5d32301d54 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:21.219877 9454 mvcc.cc:205] Tried to move back new op lower bound from 7023063454481661952 to 7023063453312122880. Current Snapshot: MvccSnapshot[applied={T|T < 7023063454481661952}]
I20240502 01:58:21.223249 9057 mvcc.cc:205] Tried to move back new op lower bound from 7023063454455025664 to 7023063453312122880. Current Snapshot: MvccSnapshot[applied={T|T < 7023063454455025664}]
I20240502 01:58:21.235224 9456 mvcc.cc:205] Tried to move back new op lower bound from 7023063454481661952 to 7023063453312122880. Current Snapshot: MvccSnapshot[applied={T|T < 7023063454481661952}]
INSERT report
rows total: 2000
time total: 112.578 ms
time per row: 0.0562892 ms
I20240502 01:58:21.252257 8088 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34536:
name: "rebalance_test_table_8"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:21.264298 8220 tablet_service.cc:1467] Processing CreateTablet for tablet 7e5754b0717940b685d5b3330598b7b0 (DEFAULT_TABLE table=rebalance_test_table_8 [id=ff6857c685a94cb6a8a5e218684726ce]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.264681 8220 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7e5754b0717940b685d5b3330598b7b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.264796 8485 tablet_service.cc:1467] Processing CreateTablet for tablet 7e5754b0717940b685d5b3330598b7b0 (DEFAULT_TABLE table=rebalance_test_table_8 [id=ff6857c685a94cb6a8a5e218684726ce]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.265084 8485 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7e5754b0717940b685d5b3330598b7b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.265185 8486 tablet_service.cc:1467] Processing CreateTablet for tablet b4ef0bb19ba24ebbaa905a2c5add37ba (DEFAULT_TABLE table=rebalance_test_table_8 [id=ff6857c685a94cb6a8a5e218684726ce]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.265455 8486 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b4ef0bb19ba24ebbaa905a2c5add37ba. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.267218 9475 tablet_bootstrap.cc:492] T b4ef0bb19ba24ebbaa905a2c5add37ba P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:21.267328 9473 tablet_bootstrap.cc:492] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:21.267952 8352 tablet_service.cc:1467] Processing CreateTablet for tablet fb34f32eaf31480abdf7145d76ed76cf (DEFAULT_TABLE table=rebalance_test_table_8 [id=ff6857c685a94cb6a8a5e218684726ce]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.268216 9475 tablet_bootstrap.cc:654] T b4ef0bb19ba24ebbaa905a2c5add37ba P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.268239 8352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fb34f32eaf31480abdf7145d76ed76cf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.268301 9473 tablet_bootstrap.cc:654] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.268337 8620 tablet_service.cc:1467] Processing CreateTablet for tablet fb34f32eaf31480abdf7145d76ed76cf (DEFAULT_TABLE table=rebalance_test_table_8 [id=ff6857c685a94cb6a8a5e218684726ce]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.268640 8620 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fb34f32eaf31480abdf7145d76ed76cf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.269903 9475 tablet_bootstrap.cc:492] T b4ef0bb19ba24ebbaa905a2c5add37ba P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:21.270009 9475 ts_tablet_manager.cc:1367] T b4ef0bb19ba24ebbaa905a2c5add37ba P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:21.270020 8756 tablet_service.cc:1467] Processing CreateTablet for tablet 7e5754b0717940b685d5b3330598b7b0 (DEFAULT_TABLE table=rebalance_test_table_8 [id=ff6857c685a94cb6a8a5e218684726ce]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.270272 8756 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7e5754b0717940b685d5b3330598b7b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.270817 9475 raft_consensus.cc:348] T b4ef0bb19ba24ebbaa905a2c5add37ba P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:21.270928 9475 raft_consensus.cc:374] T b4ef0bb19ba24ebbaa905a2c5add37ba P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.270959 9475 raft_consensus.cc:729] T b4ef0bb19ba24ebbaa905a2c5add37ba P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.270957 8891 tablet_service.cc:1467] Processing CreateTablet for tablet fb34f32eaf31480abdf7145d76ed76cf (DEFAULT_TABLE table=rebalance_test_table_8 [id=ff6857c685a94cb6a8a5e218684726ce]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.271102 9475 consensus_queue.cc:260] T b4ef0bb19ba24ebbaa905a2c5add37ba P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:21.271227 8891 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fb34f32eaf31480abdf7145d76ed76cf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.271390 9475 ts_tablet_manager.cc:1398] T b4ef0bb19ba24ebbaa905a2c5add37ba P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:21.272634 9479 tablet_bootstrap.cc:492] T 7e5754b0717940b685d5b3330598b7b0 P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:21.273512 9471 tablet_bootstrap.cc:492] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:21.273551 9475 tablet_bootstrap.cc:492] T 7e5754b0717940b685d5b3330598b7b0 P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:21.273600 9472 tablet_bootstrap.cc:492] T fb34f32eaf31480abdf7145d76ed76cf P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:21.273699 9344 tablet_bootstrap.cc:492] T fb34f32eaf31480abdf7145d76ed76cf P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:21.274219 8757 tablet_service.cc:1467] Processing CreateTablet for tablet b4ef0bb19ba24ebbaa905a2c5add37ba (DEFAULT_TABLE table=rebalance_test_table_8 [id=ff6857c685a94cb6a8a5e218684726ce]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.274446 8757 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b4ef0bb19ba24ebbaa905a2c5add37ba. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.274498 9475 tablet_bootstrap.cc:654] T 7e5754b0717940b685d5b3330598b7b0 P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.274760 9471 tablet_bootstrap.cc:654] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.274801 9344 tablet_bootstrap.cc:654] T fb34f32eaf31480abdf7145d76ed76cf P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.276263 9472 tablet_bootstrap.cc:654] T fb34f32eaf31480abdf7145d76ed76cf P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.277333 9344 tablet_bootstrap.cc:492] T fb34f32eaf31480abdf7145d76ed76cf P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:21.277446 9344 ts_tablet_manager.cc:1367] T fb34f32eaf31480abdf7145d76ed76cf P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:58:21.277632 9479 tablet_bootstrap.cc:654] T 7e5754b0717940b685d5b3330598b7b0 P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.277885 9344 raft_consensus.cc:348] T fb34f32eaf31480abdf7145d76ed76cf P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:21.277985 9344 raft_consensus.cc:374] T fb34f32eaf31480abdf7145d76ed76cf P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.278014 9344 raft_consensus.cc:729] T fb34f32eaf31480abdf7145d76ed76cf P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.278120 9344 consensus_queue.cc:260] T fb34f32eaf31480abdf7145d76ed76cf P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:21.278276 8220 tablet_service.cc:1467] Processing CreateTablet for tablet b4ef0bb19ba24ebbaa905a2c5add37ba (DEFAULT_TABLE table=rebalance_test_table_8 [id=ff6857c685a94cb6a8a5e218684726ce]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.278470 9344 ts_tablet_manager.cc:1398] T fb34f32eaf31480abdf7145d76ed76cf P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:21.279026 8220 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b4ef0bb19ba24ebbaa905a2c5add37ba. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.279492 9472 tablet_bootstrap.cc:492] T fb34f32eaf31480abdf7145d76ed76cf P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:21.279604 9472 ts_tablet_manager.cc:1367] T fb34f32eaf31480abdf7145d76ed76cf P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 01:58:21.280023 9472 raft_consensus.cc:348] T fb34f32eaf31480abdf7145d76ed76cf P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:21.280151 9472 raft_consensus.cc:374] T fb34f32eaf31480abdf7145d76ed76cf P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.280193 9472 raft_consensus.cc:729] T fb34f32eaf31480abdf7145d76ed76cf P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.280328 9472 consensus_queue.cc:260] T fb34f32eaf31480abdf7145d76ed76cf P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:21.280609 9472 ts_tablet_manager.cc:1398] T fb34f32eaf31480abdf7145d76ed76cf P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:21.281790 9473 tablet_bootstrap.cc:492] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:21.281914 9473 ts_tablet_manager.cc:1367] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.000s
I20240502 01:58:21.282366 9473 raft_consensus.cc:348] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.282480 9473 raft_consensus.cc:374] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.282513 9473 raft_consensus.cc:729] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.282773 9473 consensus_queue.cc:260] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.283071 9473 ts_tablet_manager.cc:1398] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:21.283661 9471 tablet_bootstrap.cc:492] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:21.283766 9471 ts_tablet_manager.cc:1367] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.000s
I20240502 01:58:21.284217 9471 raft_consensus.cc:348] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:21.284356 9471 raft_consensus.cc:374] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.284394 9471 raft_consensus.cc:729] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.284433 9473 tablet_bootstrap.cc:492] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:21.284519 9471 consensus_queue.cc:260] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:21.284801 9471 ts_tablet_manager.cc:1398] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:21.285609 9479 tablet_bootstrap.cc:492] T 7e5754b0717940b685d5b3330598b7b0 P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:21.285705 9479 ts_tablet_manager.cc:1367] T 7e5754b0717940b685d5b3330598b7b0 P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.013s user 0.002s sys 0.000s
I20240502 01:58:21.285713 9473 tablet_bootstrap.cc:654] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.285833 9475 tablet_bootstrap.cc:492] T 7e5754b0717940b685d5b3330598b7b0 P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:21.285923 9475 ts_tablet_manager.cc:1367] T 7e5754b0717940b685d5b3330598b7b0 P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.012s user 0.002s sys 0.000s
I20240502 01:58:21.286182 9479 raft_consensus.cc:348] T 7e5754b0717940b685d5b3330598b7b0 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.286289 9479 raft_consensus.cc:374] T 7e5754b0717940b685d5b3330598b7b0 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.286330 9479 raft_consensus.cc:729] T 7e5754b0717940b685d5b3330598b7b0 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.286343 9475 raft_consensus.cc:348] T 7e5754b0717940b685d5b3330598b7b0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.286445 9475 raft_consensus.cc:374] T 7e5754b0717940b685d5b3330598b7b0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.286478 9475 raft_consensus.cc:729] T 7e5754b0717940b685d5b3330598b7b0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.286469 9479 consensus_queue.cc:260] T 7e5754b0717940b685d5b3330598b7b0 P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.286620 9475 consensus_queue.cc:260] T 7e5754b0717940b685d5b3330598b7b0 P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.286782 9473 tablet_bootstrap.cc:492] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:21.286782 9479 ts_tablet_manager.cc:1398] T 7e5754b0717940b685d5b3330598b7b0 P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:21.286854 9473 ts_tablet_manager.cc:1367] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.002s
I20240502 01:58:21.286899 9475 ts_tablet_manager.cc:1398] T 7e5754b0717940b685d5b3330598b7b0 P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:21.286952 9479 tablet_bootstrap.cc:492] T b4ef0bb19ba24ebbaa905a2c5add37ba P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:21.287242 9473 raft_consensus.cc:348] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:21.287348 9473 raft_consensus.cc:374] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.287379 9473 raft_consensus.cc:729] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.287492 9473 consensus_queue.cc:260] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:21.287770 9473 ts_tablet_manager.cc:1398] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:21.287825 9479 tablet_bootstrap.cc:654] T b4ef0bb19ba24ebbaa905a2c5add37ba P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.290853 9479 tablet_bootstrap.cc:492] T b4ef0bb19ba24ebbaa905a2c5add37ba P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:21.290944 9479 ts_tablet_manager.cc:1367] T b4ef0bb19ba24ebbaa905a2c5add37ba P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.002s
I20240502 01:58:21.291618 9479 raft_consensus.cc:348] T b4ef0bb19ba24ebbaa905a2c5add37ba P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:21.291707 9479 raft_consensus.cc:374] T b4ef0bb19ba24ebbaa905a2c5add37ba P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.291728 9479 raft_consensus.cc:729] T b4ef0bb19ba24ebbaa905a2c5add37ba P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.291832 9479 consensus_queue.cc:260] T b4ef0bb19ba24ebbaa905a2c5add37ba P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:21.292094 9479 ts_tablet_manager.cc:1398] T b4ef0bb19ba24ebbaa905a2c5add37ba P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:21.293390 9488 raft_consensus.cc:482] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:21.293489 9488 raft_consensus.cc:504] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:21.293850 9488 leader_election.cc:288] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887), ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021)
I20240502 01:58:21.294258 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb34f32eaf31480abdf7145d76ed76cf" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" is_pre_election: true
I20240502 01:58:21.294404 8642 raft_consensus.cc:2454] T fb34f32eaf31480abdf7145d76ed76cf P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 73ee70f488334b8f8ffb83aef6456e15 in term 0.
I20240502 01:58:21.294679 8305 leader_election.cc:302] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:21.294675 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb34f32eaf31480abdf7145d76ed76cf" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" is_pre_election: true
I20240502 01:58:21.294804 8912 raft_consensus.cc:2454] T fb34f32eaf31480abdf7145d76ed76cf P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 73ee70f488334b8f8ffb83aef6456e15 in term 0.
I20240502 01:58:21.294828 9488 raft_consensus.cc:2781] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:21.294878 9488 raft_consensus.cc:482] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:21.294903 9488 raft_consensus.cc:3037] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.295807 9488 raft_consensus.cc:504] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:21.296090 9488 leader_election.cc:288] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 election: Requested vote from peers 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887), ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021)
I20240502 01:58:21.296386 8911 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb34f32eaf31480abdf7145d76ed76cf" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
I20240502 01:58:21.296506 8911 raft_consensus.cc:3037] T fb34f32eaf31480abdf7145d76ed76cf P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.296478 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb34f32eaf31480abdf7145d76ed76cf" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1"
I20240502 01:58:21.296584 8642 raft_consensus.cc:3037] T fb34f32eaf31480abdf7145d76ed76cf P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.297262 8911 raft_consensus.cc:2454] T fb34f32eaf31480abdf7145d76ed76cf P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 73ee70f488334b8f8ffb83aef6456e15 in term 1.
I20240502 01:58:21.297272 8642 raft_consensus.cc:2454] T fb34f32eaf31480abdf7145d76ed76cf P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 73ee70f488334b8f8ffb83aef6456e15 in term 1.
I20240502 01:58:21.297541 8305 leader_election.cc:302] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, 73ee70f488334b8f8ffb83aef6456e15; no voters:
I20240502 01:58:21.297739 9488 raft_consensus.cc:2781] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:21.297825 9488 raft_consensus.cc:686] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [term 1 LEADER]: Becoming Leader. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Running, Role: LEADER
I20240502 01:58:21.297938 9488 consensus_queue.cc:237] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:21.299564 8093 catalog_manager.cc:5484] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 reported cstate change: term changed from 0 to 1, leader changed from <none> to 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "73ee70f488334b8f8ffb83aef6456e15" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:21.345070 9341 raft_consensus.cc:482] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:21.345223 9341 raft_consensus.cc:504] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:21.345680 9341 leader_election.cc:288] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775), e325acceaed541869d43c1f762530c2e (127.7.221.67:42267)
I20240502 01:58:21.346122 8776 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b4ef0bb19ba24ebbaa905a2c5add37ba" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54" is_pre_election: true
I20240502 01:58:21.346133 8506 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b4ef0bb19ba24ebbaa905a2c5add37ba" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e" is_pre_election: true
I20240502 01:58:21.346259 8776 raft_consensus.cc:2454] T b4ef0bb19ba24ebbaa905a2c5add37ba P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 0.
I20240502 01:58:21.346274 8506 raft_consensus.cc:2454] T b4ef0bb19ba24ebbaa905a2c5add37ba P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 0.
I20240502 01:58:21.346563 8172 leader_election.cc:302] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:21.346752 9341 raft_consensus.cc:2781] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:21.346818 9341 raft_consensus.cc:482] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:21.346858 9341 raft_consensus.cc:3037] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.347633 9341 raft_consensus.cc:504] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:21.347970 9341 leader_election.cc:288] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 election: Requested vote from peers 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775), e325acceaed541869d43c1f762530c2e (127.7.221.67:42267)
I20240502 01:58:21.348282 8776 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b4ef0bb19ba24ebbaa905a2c5add37ba" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54"
I20240502 01:58:21.348336 8506 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b4ef0bb19ba24ebbaa905a2c5add37ba" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e"
I20240502 01:58:21.348392 8776 raft_consensus.cc:3037] T b4ef0bb19ba24ebbaa905a2c5add37ba P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.348435 8506 raft_consensus.cc:3037] T b4ef0bb19ba24ebbaa905a2c5add37ba P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.349120 8506 raft_consensus.cc:2454] T b4ef0bb19ba24ebbaa905a2c5add37ba P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 1.
I20240502 01:58:21.349129 8776 raft_consensus.cc:2454] T b4ef0bb19ba24ebbaa905a2c5add37ba P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 1.
I20240502 01:58:21.349426 8173 leader_election.cc:302] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e325acceaed541869d43c1f762530c2e, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:21.349571 9341 raft_consensus.cc:2781] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:21.349731 9341 raft_consensus.cc:686] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 LEADER]: Becoming Leader. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Running, Role: LEADER
I20240502 01:58:21.349875 9341 consensus_queue.cc:237] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:21.351644 8093 catalog_manager.cc:5484] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 reported cstate change: term changed from 0 to 1, leader changed from <none> to e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:21.555577 9341 raft_consensus.cc:482] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:21.555702 9341 raft_consensus.cc:504] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.556157 9341 leader_election.cc:288] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e325acceaed541869d43c1f762530c2e (127.7.221.67:42267), 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775)
I20240502 01:58:21.556527 8506 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e5754b0717940b685d5b3330598b7b0" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e" is_pre_election: true
I20240502 01:58:21.556599 8776 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e5754b0717940b685d5b3330598b7b0" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54" is_pre_election: true
I20240502 01:58:21.556692 8506 raft_consensus.cc:2454] T 7e5754b0717940b685d5b3330598b7b0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 0.
I20240502 01:58:21.556785 8776 raft_consensus.cc:2454] T 7e5754b0717940b685d5b3330598b7b0 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 0.
I20240502 01:58:21.556984 8173 leader_election.cc:302] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e325acceaed541869d43c1f762530c2e, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:21.557152 9341 raft_consensus.cc:2781] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:21.557219 9341 raft_consensus.cc:482] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:21.557252 9341 raft_consensus.cc:3037] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.558095 9341 raft_consensus.cc:504] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.558387 9341 leader_election.cc:288] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 election: Requested vote from peers e325acceaed541869d43c1f762530c2e (127.7.221.67:42267), 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775)
I20240502 01:58:21.558705 8506 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e5754b0717940b685d5b3330598b7b0" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e"
I20240502 01:58:21.558858 8506 raft_consensus.cc:3037] T 7e5754b0717940b685d5b3330598b7b0 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.558852 8776 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e5754b0717940b685d5b3330598b7b0" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54"
I20240502 01:58:21.558956 8776 raft_consensus.cc:3037] T 7e5754b0717940b685d5b3330598b7b0 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.559803 8506 raft_consensus.cc:2454] T 7e5754b0717940b685d5b3330598b7b0 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 1.
I20240502 01:58:21.559820 8776 raft_consensus.cc:2454] T 7e5754b0717940b685d5b3330598b7b0 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 1.
I20240502 01:58:21.560071 8173 leader_election.cc:302] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e325acceaed541869d43c1f762530c2e, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:21.560346 9341 raft_consensus.cc:2781] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:21.560470 9341 raft_consensus.cc:686] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 LEADER]: Becoming Leader. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Running, Role: LEADER
I20240502 01:58:21.560621 9341 consensus_queue.cc:237] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.562429 8093 catalog_manager.cc:5484] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 reported cstate change: term changed from 0 to 1, leader changed from <none> to e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: HEALTHY } } }
W20240502 01:58:21.677955 9546 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:21.678186 9546 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:21.681474 9546 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_8'
I20240502 01:58:21.705387 9325 consensus_queue.cc:1035] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:21.719261 9488 consensus_queue.cc:1035] T fb34f32eaf31480abdf7145d76ed76cf P 73ee70f488334b8f8ffb83aef6456e15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:21.773262 8506 raft_consensus.cc:1261] T 7e5754b0717940b685d5b3330598b7b0 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Refusing update from remote peer e7a8991bb9ff47489bf8ccbaf5ca7fa9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:21.773972 9341 consensus_queue.cc:1035] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:21.775337 8776 raft_consensus.cc:1261] T 7e5754b0717940b685d5b3330598b7b0 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Refusing update from remote peer e7a8991bb9ff47489bf8ccbaf5ca7fa9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:21.775672 8777 raft_consensus.cc:1261] T b4ef0bb19ba24ebbaa905a2c5add37ba P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Refusing update from remote peer e7a8991bb9ff47489bf8ccbaf5ca7fa9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:21.779412 9543 consensus_queue.cc:1035] T 7e5754b0717940b685d5b3330598b7b0 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:21.779453 9582 consensus_queue.cc:1035] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:21.784263 8506 raft_consensus.cc:1261] T b4ef0bb19ba24ebbaa905a2c5add37ba P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Refusing update from remote peer e7a8991bb9ff47489bf8ccbaf5ca7fa9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:21.784698 9341 consensus_queue.cc:1035] T b4ef0bb19ba24ebbaa905a2c5add37ba P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 99.6043 ms
time per row: 0.0498022 ms
I20240502 01:58:21.834561 8091 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34536:
name: "rebalance_test_table_9"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:21.840351 8620 tablet_service.cc:1467] Processing CreateTablet for tablet 9f467a9793204cedb6c74004423d8e80 (DEFAULT_TABLE table=rebalance_test_table_9 [id=00ffb50e157c438eb6d6fb8eebb9698c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.840713 8620 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f467a9793204cedb6c74004423d8e80. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.841089 8485 tablet_service.cc:1467] Processing CreateTablet for tablet 2ebb407657b94fb5ae587288859e40fa (DEFAULT_TABLE table=rebalance_test_table_9 [id=00ffb50e157c438eb6d6fb8eebb9698c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.841331 8485 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ebb407657b94fb5ae587288859e40fa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.841756 8220 tablet_service.cc:1467] Processing CreateTablet for tablet 2ebb407657b94fb5ae587288859e40fa (DEFAULT_TABLE table=rebalance_test_table_9 [id=00ffb50e157c438eb6d6fb8eebb9698c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.842092 8220 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ebb407657b94fb5ae587288859e40fa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.842685 9611 tablet_bootstrap.cc:492] T 9f467a9793204cedb6c74004423d8e80 P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:21.842898 8891 tablet_service.cc:1467] Processing CreateTablet for tablet 9f467a9793204cedb6c74004423d8e80 (DEFAULT_TABLE table=rebalance_test_table_9 [id=00ffb50e157c438eb6d6fb8eebb9698c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.843191 8891 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f467a9793204cedb6c74004423d8e80. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.843582 9611 tablet_bootstrap.cc:654] T 9f467a9793204cedb6c74004423d8e80 P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.844053 9612 tablet_bootstrap.cc:492] T 2ebb407657b94fb5ae587288859e40fa P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:21.844741 9612 tablet_bootstrap.cc:654] T 2ebb407657b94fb5ae587288859e40fa P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.845175 9614 tablet_bootstrap.cc:492] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:21.845382 8352 tablet_service.cc:1467] Processing CreateTablet for tablet 9f467a9793204cedb6c74004423d8e80 (DEFAULT_TABLE table=rebalance_test_table_9 [id=00ffb50e157c438eb6d6fb8eebb9698c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.845671 8352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f467a9793204cedb6c74004423d8e80. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.845978 9614 tablet_bootstrap.cc:654] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.846004 9612 tablet_bootstrap.cc:492] T 2ebb407657b94fb5ae587288859e40fa P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:21.846077 9612 ts_tablet_manager.cc:1367] T 2ebb407657b94fb5ae587288859e40fa P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:21.846549 9612 raft_consensus.cc:348] T 2ebb407657b94fb5ae587288859e40fa P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.846769 9612 raft_consensus.cc:374] T 2ebb407657b94fb5ae587288859e40fa P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.846808 9612 raft_consensus.cc:729] T 2ebb407657b94fb5ae587288859e40fa P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.846947 9612 consensus_queue.cc:260] T 2ebb407657b94fb5ae587288859e40fa P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.847198 9612 ts_tablet_manager.cc:1398] T 2ebb407657b94fb5ae587288859e40fa P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:21.847469 9617 tablet_bootstrap.cc:492] T 9f467a9793204cedb6c74004423d8e80 P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:21.847649 8892 tablet_service.cc:1467] Processing CreateTablet for tablet ffe9b58784cd4a60993e632ebd3bd32f (DEFAULT_TABLE table=rebalance_test_table_9 [id=00ffb50e157c438eb6d6fb8eebb9698c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.847883 8892 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ffe9b58784cd4a60993e632ebd3bd32f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.848260 8351 tablet_service.cc:1467] Processing CreateTablet for tablet ffe9b58784cd4a60993e632ebd3bd32f (DEFAULT_TABLE table=rebalance_test_table_9 [id=00ffb50e157c438eb6d6fb8eebb9698c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.848392 9617 tablet_bootstrap.cc:654] T 9f467a9793204cedb6c74004423d8e80 P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.848472 8351 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ffe9b58784cd4a60993e632ebd3bd32f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.849704 9617 tablet_bootstrap.cc:492] T 9f467a9793204cedb6c74004423d8e80 P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:21.849792 9617 ts_tablet_manager.cc:1367] T 9f467a9793204cedb6c74004423d8e80 P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:21.850514 8757 tablet_service.cc:1467] Processing CreateTablet for tablet 2ebb407657b94fb5ae587288859e40fa (DEFAULT_TABLE table=rebalance_test_table_9 [id=00ffb50e157c438eb6d6fb8eebb9698c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.850697 8621 tablet_service.cc:1467] Processing CreateTablet for tablet ffe9b58784cd4a60993e632ebd3bd32f (DEFAULT_TABLE table=rebalance_test_table_9 [id=00ffb50e157c438eb6d6fb8eebb9698c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:21.850790 8757 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ebb407657b94fb5ae587288859e40fa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.850914 8621 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ffe9b58784cd4a60993e632ebd3bd32f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:21.852505 9619 tablet_bootstrap.cc:492] T 2ebb407657b94fb5ae587288859e40fa P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:21.850224 9617 raft_consensus.cc:348] T 9f467a9793204cedb6c74004423d8e80 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:21.852572 9617 raft_consensus.cc:374] T 9f467a9793204cedb6c74004423d8e80 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.852655 9617 raft_consensus.cc:729] T 9f467a9793204cedb6c74004423d8e80 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.852829 9617 consensus_queue.cc:260] T 9f467a9793204cedb6c74004423d8e80 P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:21.853142 9617 ts_tablet_manager.cc:1398] T 9f467a9793204cedb6c74004423d8e80 P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:21.853294 9617 tablet_bootstrap.cc:492] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:21.853446 9619 tablet_bootstrap.cc:654] T 2ebb407657b94fb5ae587288859e40fa P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.854202 9617 tablet_bootstrap.cc:654] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.854622 9619 tablet_bootstrap.cc:492] T 2ebb407657b94fb5ae587288859e40fa P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:21.854704 9619 ts_tablet_manager.cc:1367] T 2ebb407657b94fb5ae587288859e40fa P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:21.854733 9611 tablet_bootstrap.cc:492] T 9f467a9793204cedb6c74004423d8e80 P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:21.854820 9611 ts_tablet_manager.cc:1367] T 9f467a9793204cedb6c74004423d8e80 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.012s user 0.002s sys 0.000s
I20240502 01:58:21.855108 9619 raft_consensus.cc:348] T 2ebb407657b94fb5ae587288859e40fa P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.855206 9619 raft_consensus.cc:374] T 2ebb407657b94fb5ae587288859e40fa P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.855159 9611 raft_consensus.cc:348] T 9f467a9793204cedb6c74004423d8e80 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:21.855233 9619 raft_consensus.cc:729] T 2ebb407657b94fb5ae587288859e40fa P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.855244 9611 raft_consensus.cc:374] T 9f467a9793204cedb6c74004423d8e80 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.855275 9611 raft_consensus.cc:729] T 9f467a9793204cedb6c74004423d8e80 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.855360 9619 consensus_queue.cc:260] T 2ebb407657b94fb5ae587288859e40fa P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.855381 9611 consensus_queue.cc:260] T 9f467a9793204cedb6c74004423d8e80 P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:21.855643 9611 ts_tablet_manager.cc:1398] T 9f467a9793204cedb6c74004423d8e80 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:21.855643 9619 ts_tablet_manager.cc:1398] T 2ebb407657b94fb5ae587288859e40fa P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:21.855818 9611 tablet_bootstrap.cc:492] T ffe9b58784cd4a60993e632ebd3bd32f P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:21.856601 9621 tablet_bootstrap.cc:492] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:21.856653 9611 tablet_bootstrap.cc:654] T ffe9b58784cd4a60993e632ebd3bd32f P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.857466 9617 tablet_bootstrap.cc:492] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:21.857537 9617 ts_tablet_manager.cc:1367] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.002s
I20240502 01:58:21.857962 9617 raft_consensus.cc:348] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:21.858062 9617 raft_consensus.cc:374] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.858103 9617 raft_consensus.cc:729] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.858217 9617 consensus_queue.cc:260] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:21.858491 9617 ts_tablet_manager.cc:1398] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:21.859287 9621 tablet_bootstrap.cc:654] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.860857 9621 tablet_bootstrap.cc:492] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:21.860931 9621 ts_tablet_manager.cc:1367] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:58:21.861276 9621 raft_consensus.cc:348] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.861359 9621 raft_consensus.cc:374] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.861392 9621 raft_consensus.cc:729] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.861514 9621 consensus_queue.cc:260] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.861753 9621 ts_tablet_manager.cc:1398] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:21.870939 9611 tablet_bootstrap.cc:492] T ffe9b58784cd4a60993e632ebd3bd32f P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:21.871001 9614 tablet_bootstrap.cc:492] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:21.871037 9611 ts_tablet_manager.cc:1367] T ffe9b58784cd4a60993e632ebd3bd32f P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.000s
I20240502 01:58:21.871104 9614 ts_tablet_manager.cc:1367] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.026s user 0.002s sys 0.000s
I20240502 01:58:21.871490 9611 raft_consensus.cc:348] T ffe9b58784cd4a60993e632ebd3bd32f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:21.871541 9614 raft_consensus.cc:348] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:21.871598 9611 raft_consensus.cc:374] T ffe9b58784cd4a60993e632ebd3bd32f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.871624 9614 raft_consensus.cc:374] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.871627 9611 raft_consensus.cc:729] T ffe9b58784cd4a60993e632ebd3bd32f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.871644 9614 raft_consensus.cc:729] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.871760 9611 consensus_queue.cc:260] T ffe9b58784cd4a60993e632ebd3bd32f P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:21.871788 9614 consensus_queue.cc:260] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:21.872116 9614 ts_tablet_manager.cc:1398] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:21.872241 9611 ts_tablet_manager.cc:1398] T ffe9b58784cd4a60993e632ebd3bd32f P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:21.874058 9614 tablet_bootstrap.cc:492] T ffe9b58784cd4a60993e632ebd3bd32f P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:21.875038 9614 tablet_bootstrap.cc:654] T ffe9b58784cd4a60993e632ebd3bd32f P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:21.876427 9614 tablet_bootstrap.cc:492] T ffe9b58784cd4a60993e632ebd3bd32f P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:21.876513 9614 ts_tablet_manager.cc:1367] T ffe9b58784cd4a60993e632ebd3bd32f P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:21.876896 9614 raft_consensus.cc:348] T ffe9b58784cd4a60993e632ebd3bd32f P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:21.876997 9614 raft_consensus.cc:374] T ffe9b58784cd4a60993e632ebd3bd32f P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:21.877030 9614 raft_consensus.cc:729] T ffe9b58784cd4a60993e632ebd3bd32f P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:21.877180 9614 consensus_queue.cc:260] T ffe9b58784cd4a60993e632ebd3bd32f P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:21.877444 9614 ts_tablet_manager.cc:1398] T ffe9b58784cd4a60993e632ebd3bd32f P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:21.946080 9584 raft_consensus.cc:482] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:21.946234 9584 raft_consensus.cc:504] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:21.946659 9584 leader_election.cc:288] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:21.947149 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ffe9b58784cd4a60993e632ebd3bd32f" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" is_pre_election: true
I20240502 01:58:21.947149 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ffe9b58784cd4a60993e632ebd3bd32f" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" is_pre_election: true
I20240502 01:58:21.947299 8912 raft_consensus.cc:2454] T ffe9b58784cd4a60993e632ebd3bd32f P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 73ee70f488334b8f8ffb83aef6456e15 in term 0.
I20240502 01:58:21.947299 8642 raft_consensus.cc:2454] T ffe9b58784cd4a60993e632ebd3bd32f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 73ee70f488334b8f8ffb83aef6456e15 in term 0.
I20240502 01:58:21.947609 8305 leader_election.cc:302] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:21.947789 9584 raft_consensus.cc:2781] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:21.947847 9584 raft_consensus.cc:482] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:21.947897 9584 raft_consensus.cc:3037] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.948765 9584 raft_consensus.cc:504] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:21.949066 9584 leader_election.cc:288] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 election: Requested vote from peers ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:21.949435 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ffe9b58784cd4a60993e632ebd3bd32f" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
I20240502 01:58:21.949433 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ffe9b58784cd4a60993e632ebd3bd32f" candidate_uuid: "73ee70f488334b8f8ffb83aef6456e15" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1"
I20240502 01:58:21.949554 8912 raft_consensus.cc:3037] T ffe9b58784cd4a60993e632ebd3bd32f P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.949554 8642 raft_consensus.cc:3037] T ffe9b58784cd4a60993e632ebd3bd32f P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.950480 8642 raft_consensus.cc:2454] T ffe9b58784cd4a60993e632ebd3bd32f P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 73ee70f488334b8f8ffb83aef6456e15 in term 1.
I20240502 01:58:21.950480 8912 raft_consensus.cc:2454] T ffe9b58784cd4a60993e632ebd3bd32f P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 73ee70f488334b8f8ffb83aef6456e15 in term 1.
I20240502 01:58:21.950798 8305 leader_election.cc:302] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:21.950945 9584 raft_consensus.cc:2781] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:21.951033 9584 raft_consensus.cc:686] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [term 1 LEADER]: Becoming Leader. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Running, Role: LEADER
I20240502 01:58:21.951181 9584 consensus_queue.cc:237] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:21.953217 8090 catalog_manager.cc:5484] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 reported cstate change: term changed from 0 to 1, leader changed from <none> to 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "73ee70f488334b8f8ffb83aef6456e15" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: HEALTHY } } }
I20240502 01:58:21.993608 9267 raft_consensus.cc:482] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:21.993752 9267 raft_consensus.cc:504] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.994179 9267 leader_election.cc:288] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775), e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919)
I20240502 01:58:21.994576 8776 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ebb407657b94fb5ae587288859e40fa" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54" is_pre_election: true
I20240502 01:58:21.994633 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ebb407657b94fb5ae587288859e40fa" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" is_pre_election: true
I20240502 01:58:21.994743 8776 raft_consensus.cc:2454] T 2ebb407657b94fb5ae587288859e40fa P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 0.
I20240502 01:58:21.994827 8240 raft_consensus.cc:2454] T 2ebb407657b94fb5ae587288859e40fa P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 0.
I20240502 01:58:21.995019 8439 leader_election.cc:302] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e325acceaed541869d43c1f762530c2e; no voters:
I20240502 01:58:21.995199 9267 raft_consensus.cc:2781] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:21.995263 9267 raft_consensus.cc:482] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:21.995287 9267 raft_consensus.cc:3037] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.996057 9267 raft_consensus.cc:504] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:21.996349 9267 leader_election.cc:288] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 election: Requested vote from peers 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775), e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919)
I20240502 01:58:21.996631 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ebb407657b94fb5ae587288859e40fa" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
I20240502 01:58:21.996632 8776 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ebb407657b94fb5ae587288859e40fa" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54"
I20240502 01:58:21.996734 8240 raft_consensus.cc:3037] T 2ebb407657b94fb5ae587288859e40fa P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.996752 8776 raft_consensus.cc:3037] T 2ebb407657b94fb5ae587288859e40fa P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:21.997493 8776 raft_consensus.cc:2454] T 2ebb407657b94fb5ae587288859e40fa P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 1.
I20240502 01:58:21.997501 8240 raft_consensus.cc:2454] T 2ebb407657b94fb5ae587288859e40fa P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 1.
I20240502 01:58:21.997754 8439 leader_election.cc:302] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e325acceaed541869d43c1f762530c2e; no voters:
I20240502 01:58:21.997923 9267 raft_consensus.cc:2781] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:21.998131 9267 raft_consensus.cc:686] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [term 1 LEADER]: Becoming Leader. State: Replica: e325acceaed541869d43c1f762530c2e, State: Running, Role: LEADER
I20240502 01:58:21.998276 9267 consensus_queue.cc:237] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } }
I20240502 01:58:22.000097 8090 catalog_manager.cc:5484] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e reported cstate change: term changed from 0 to 1, leader changed from <none> to e325acceaed541869d43c1f762530c2e (127.7.221.67). New cstate: current_term: 1 leader_uuid: "e325acceaed541869d43c1f762530c2e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:22.222088 9529 raft_consensus.cc:482] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:22.222234 9529 raft_consensus.cc:504] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:22.222688 9529 leader_election.cc:288] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021), 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503)
I20240502 01:58:22.223057 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f467a9793204cedb6c74004423d8e80" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" is_pre_election: true
I20240502 01:58:22.223083 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f467a9793204cedb6c74004423d8e80" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15" is_pre_election: true
I20240502 01:58:22.223196 8642 raft_consensus.cc:2454] T 9f467a9793204cedb6c74004423d8e80 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 0.
I20240502 01:58:22.223234 8373 raft_consensus.cc:2454] T 9f467a9793204cedb6c74004423d8e80 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 0.
I20240502 01:58:22.223497 8844 leader_election.cc:302] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:22.223664 9529 raft_consensus.cc:2781] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:22.223721 9529 raft_consensus.cc:482] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:22.223745 9529 raft_consensus.cc:3037] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:22.224525 9529 raft_consensus.cc:504] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:22.224828 9529 leader_election.cc:288] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 election: Requested vote from peers ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021), 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503)
I20240502 01:58:22.225109 8642 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f467a9793204cedb6c74004423d8e80" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1"
I20240502 01:58:22.225167 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f467a9793204cedb6c74004423d8e80" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15"
I20240502 01:58:22.225272 8373 raft_consensus.cc:3037] T 9f467a9793204cedb6c74004423d8e80 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:22.225281 8642 raft_consensus.cc:3037] T 9f467a9793204cedb6c74004423d8e80 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:22.226048 8373 raft_consensus.cc:2454] T 9f467a9793204cedb6c74004423d8e80 P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 1.
I20240502 01:58:22.226086 8642 raft_consensus.cc:2454] T 9f467a9793204cedb6c74004423d8e80 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 1.
I20240502 01:58:22.226388 8845 leader_election.cc:302] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, 73ee70f488334b8f8ffb83aef6456e15; no voters:
I20240502 01:58:22.226567 9529 raft_consensus.cc:2781] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:22.226756 9529 raft_consensus.cc:686] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 LEADER]: Becoming Leader. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Running, Role: LEADER
I20240502 01:58:22.226909 9529 consensus_queue.cc:237] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:22.228678 8090 catalog_manager.cc:5484] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb reported cstate change: term changed from 0 to 1, leader changed from <none> to 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70). New cstate: current_term: 1 leader_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:22.417096 9632 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:22.417379 9632 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:22.420477 9632 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_9'
I20240502 01:58:22.515357 8240 raft_consensus.cc:1261] T 2ebb407657b94fb5ae587288859e40fa P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Refusing update from remote peer e325acceaed541869d43c1f762530c2e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:22.515964 8642 raft_consensus.cc:1261] T ffe9b58784cd4a60993e632ebd3bd32f P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Refusing update from remote peer 73ee70f488334b8f8ffb83aef6456e15: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:22.515990 8776 raft_consensus.cc:1261] T 2ebb407657b94fb5ae587288859e40fa P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Refusing update from remote peer e325acceaed541869d43c1f762530c2e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:22.516196 8912 raft_consensus.cc:1261] T ffe9b58784cd4a60993e632ebd3bd32f P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Refusing update from remote peer 73ee70f488334b8f8ffb83aef6456e15: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:22.516674 9584 consensus_queue.cc:1035] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:22.516836 9267 consensus_queue.cc:1035] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [LEADER]: Connected to new peer: Peer: permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:22.516868 9667 consensus_queue.cc:1035] T ffe9b58784cd4a60993e632ebd3bd32f P 73ee70f488334b8f8ffb83aef6456e15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:22.517108 9267 consensus_queue.cc:1035] T 2ebb407657b94fb5ae587288859e40fa P e325acceaed541869d43c1f762530c2e [LEADER]: Connected to new peer: Peer: permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:22.522861 8373 raft_consensus.cc:1261] T 9f467a9793204cedb6c74004423d8e80 P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Refusing update from remote peer 1caeef741efa452fa9bb1fd3d47e65eb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:22.523015 8641 raft_consensus.cc:1261] T 9f467a9793204cedb6c74004423d8e80 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Refusing update from remote peer 1caeef741efa452fa9bb1fd3d47e65eb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:22.523418 9529 consensus_queue.cc:1035] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [LEADER]: Connected to new peer: Peer: permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:22.523787 9529 consensus_queue.cc:1035] T 9f467a9793204cedb6c74004423d8e80 P 1caeef741efa452fa9bb1fd3d47e65eb [LEADER]: Connected to new peer: Peer: permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:22.550094 9671 mvcc.cc:205] Tried to move back new op lower bound from 7023063459926491136 to 7023063458722975744. Current Snapshot: MvccSnapshot[applied={T|T < 7023063459926491136}]
INSERT report
rows total: 2000
time total: 122.469 ms
time per row: 0.0612345 ms
I20240502 01:58:22.588088 8093 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34536:
name: "rebalance_test_table_10"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:22.593490 8485 tablet_service.cc:1467] Processing CreateTablet for tablet 08022c1e2dee4aefaf0e39645048a512 (DEFAULT_TABLE table=rebalance_test_table_10 [id=d8ecc3997593488ebf11fb822cd37f46]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:22.593695 8220 tablet_service.cc:1467] Processing CreateTablet for tablet 08022c1e2dee4aefaf0e39645048a512 (DEFAULT_TABLE table=rebalance_test_table_10 [id=d8ecc3997593488ebf11fb822cd37f46]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:22.593822 8485 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 08022c1e2dee4aefaf0e39645048a512. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:22.594038 8220 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 08022c1e2dee4aefaf0e39645048a512. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:22.594874 8756 tablet_service.cc:1467] Processing CreateTablet for tablet d913e857eb814ccb95455fbc94272b1f (DEFAULT_TABLE table=rebalance_test_table_10 [id=d8ecc3997593488ebf11fb822cd37f46]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:22.595161 8756 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d913e857eb814ccb95455fbc94272b1f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:22.595172 8892 tablet_service.cc:1467] Processing CreateTablet for tablet e7202e760b1b41d6bdbbdea433baf1d3 (DEFAULT_TABLE table=rebalance_test_table_10 [id=d8ecc3997593488ebf11fb822cd37f46]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:22.595443 8892 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e7202e760b1b41d6bdbbdea433baf1d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:22.596498 8219 tablet_service.cc:1467] Processing CreateTablet for tablet d913e857eb814ccb95455fbc94272b1f (DEFAULT_TABLE table=rebalance_test_table_10 [id=d8ecc3997593488ebf11fb822cd37f46]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:22.596501 8757 tablet_service.cc:1467] Processing CreateTablet for tablet 08022c1e2dee4aefaf0e39645048a512 (DEFAULT_TABLE table=rebalance_test_table_10 [id=d8ecc3997593488ebf11fb822cd37f46]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:22.596704 8757 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 08022c1e2dee4aefaf0e39645048a512. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:22.596724 8219 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d913e857eb814ccb95455fbc94272b1f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:22.597445 9704 tablet_bootstrap.cc:492] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:22.598318 9704 tablet_bootstrap.cc:654] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:22.598726 8351 tablet_service.cc:1467] Processing CreateTablet for tablet e7202e760b1b41d6bdbbdea433baf1d3 (DEFAULT_TABLE table=rebalance_test_table_10 [id=d8ecc3997593488ebf11fb822cd37f46]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:22.598871 8620 tablet_service.cc:1467] Processing CreateTablet for tablet e7202e760b1b41d6bdbbdea433baf1d3 (DEFAULT_TABLE table=rebalance_test_table_10 [id=d8ecc3997593488ebf11fb822cd37f46]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:22.599007 8351 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e7202e760b1b41d6bdbbdea433baf1d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:22.599134 8620 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e7202e760b1b41d6bdbbdea433baf1d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:22.599314 9704 tablet_bootstrap.cc:492] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:22.599395 9704 ts_tablet_manager.cc:1367] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:22.599774 8485 tablet_service.cc:1467] Processing CreateTablet for tablet d913e857eb814ccb95455fbc94272b1f (DEFAULT_TABLE table=rebalance_test_table_10 [id=d8ecc3997593488ebf11fb822cd37f46]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:22.599828 9704 raft_consensus.cc:348] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.599918 9704 raft_consensus.cc:374] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:22.599954 9704 raft_consensus.cc:729] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:22.600018 8485 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d913e857eb814ccb95455fbc94272b1f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:22.600098 9704 consensus_queue.cc:260] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.600358 9704 ts_tablet_manager.cc:1398] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:22.600515 9704 tablet_bootstrap.cc:492] T 08022c1e2dee4aefaf0e39645048a512 P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:22.601507 9704 tablet_bootstrap.cc:654] T 08022c1e2dee4aefaf0e39645048a512 P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:22.601634 9703 tablet_bootstrap.cc:492] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:22.601778 9707 tablet_bootstrap.cc:492] T e7202e760b1b41d6bdbbdea433baf1d3 P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:22.602579 9705 tablet_bootstrap.cc:492] T e7202e760b1b41d6bdbbdea433baf1d3 P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:22.603041 9707 tablet_bootstrap.cc:654] T e7202e760b1b41d6bdbbdea433baf1d3 P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:22.603377 9704 tablet_bootstrap.cc:492] T 08022c1e2dee4aefaf0e39645048a512 P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:22.603482 9704 ts_tablet_manager.cc:1367] T 08022c1e2dee4aefaf0e39645048a512 P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:22.603750 9708 tablet_bootstrap.cc:492] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:22.604697 9703 tablet_bootstrap.cc:654] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:22.604629 9704 raft_consensus.cc:348] T 08022c1e2dee4aefaf0e39645048a512 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.604753 9708 tablet_bootstrap.cc:654] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:22.604768 9704 raft_consensus.cc:374] T 08022c1e2dee4aefaf0e39645048a512 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:22.604811 9704 raft_consensus.cc:729] T 08022c1e2dee4aefaf0e39645048a512 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:22.605078 9704 consensus_queue.cc:260] T 08022c1e2dee4aefaf0e39645048a512 P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.605468 9704 ts_tablet_manager.cc:1398] T 08022c1e2dee4aefaf0e39645048a512 P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:22.605515 9702 tablet_bootstrap.cc:492] T 08022c1e2dee4aefaf0e39645048a512 P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:22.606432 9705 tablet_bootstrap.cc:654] T e7202e760b1b41d6bdbbdea433baf1d3 P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:22.606427 9702 tablet_bootstrap.cc:654] T 08022c1e2dee4aefaf0e39645048a512 P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:22.606909 9708 tablet_bootstrap.cc:492] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:22.607005 9708 ts_tablet_manager.cc:1367] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:22.607579 9708 raft_consensus.cc:348] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:22.607695 9708 raft_consensus.cc:374] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:22.607731 9708 raft_consensus.cc:729] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:22.607806 9707 tablet_bootstrap.cc:492] T e7202e760b1b41d6bdbbdea433baf1d3 P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:22.607837 9705 tablet_bootstrap.cc:492] T e7202e760b1b41d6bdbbdea433baf1d3 P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:22.607903 9707 ts_tablet_manager.cc:1367] T e7202e760b1b41d6bdbbdea433baf1d3 P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 01:58:22.607916 9705 ts_tablet_manager.cc:1367] T e7202e760b1b41d6bdbbdea433baf1d3 P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.002s
I20240502 01:58:22.607875 9708 consensus_queue.cc:260] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:22.608361 9705 raft_consensus.cc:348] T e7202e760b1b41d6bdbbdea433baf1d3 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:22.608407 9707 raft_consensus.cc:348] T e7202e760b1b41d6bdbbdea433baf1d3 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:22.608479 9705 raft_consensus.cc:374] T e7202e760b1b41d6bdbbdea433baf1d3 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:22.608501 9707 raft_consensus.cc:374] T e7202e760b1b41d6bdbbdea433baf1d3 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:22.608513 9705 raft_consensus.cc:729] T e7202e760b1b41d6bdbbdea433baf1d3 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:22.608544 9707 raft_consensus.cc:729] T e7202e760b1b41d6bdbbdea433baf1d3 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:22.608659 9708 ts_tablet_manager.cc:1398] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:22.608630 9705 consensus_queue.cc:260] T e7202e760b1b41d6bdbbdea433baf1d3 P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:22.608664 9707 consensus_queue.cc:260] T e7202e760b1b41d6bdbbdea433baf1d3 P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:22.608891 9705 ts_tablet_manager.cc:1398] T e7202e760b1b41d6bdbbdea433baf1d3 P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:22.608992 9707 ts_tablet_manager.cc:1398] T e7202e760b1b41d6bdbbdea433baf1d3 P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:22.609973 9703 tablet_bootstrap.cc:492] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:22.610147 9703 ts_tablet_manager.cc:1367] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.009s user 0.002s sys 0.000s
I20240502 01:58:22.610704 9703 raft_consensus.cc:348] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.610867 9703 raft_consensus.cc:374] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:22.610966 9703 raft_consensus.cc:729] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:22.611160 9703 consensus_queue.cc:260] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.611503 9703 ts_tablet_manager.cc:1398] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:22.612221 9702 tablet_bootstrap.cc:492] T 08022c1e2dee4aefaf0e39645048a512 P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:22.612370 9702 ts_tablet_manager.cc:1367] T 08022c1e2dee4aefaf0e39645048a512 P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240502 01:58:22.612437 9703 tablet_bootstrap.cc:492] T d913e857eb814ccb95455fbc94272b1f P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:22.613296 9703 tablet_bootstrap.cc:654] T d913e857eb814ccb95455fbc94272b1f P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:22.614305 9703 tablet_bootstrap.cc:492] T d913e857eb814ccb95455fbc94272b1f P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:22.614318 9702 raft_consensus.cc:348] T 08022c1e2dee4aefaf0e39645048a512 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.614377 9703 ts_tablet_manager.cc:1367] T d913e857eb814ccb95455fbc94272b1f P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:22.614401 9702 raft_consensus.cc:374] T 08022c1e2dee4aefaf0e39645048a512 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:22.614421 9702 raft_consensus.cc:729] T 08022c1e2dee4aefaf0e39645048a512 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:22.614517 9702 consensus_queue.cc:260] T 08022c1e2dee4aefaf0e39645048a512 P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.614796 9703 raft_consensus.cc:348] T d913e857eb814ccb95455fbc94272b1f P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.614888 9703 raft_consensus.cc:374] T d913e857eb814ccb95455fbc94272b1f P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:22.614918 9703 raft_consensus.cc:729] T d913e857eb814ccb95455fbc94272b1f P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:22.614936 9702 ts_tablet_manager.cc:1398] T 08022c1e2dee4aefaf0e39645048a512 P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:22.615015 9703 consensus_queue.cc:260] T d913e857eb814ccb95455fbc94272b1f P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.615166 9702 tablet_bootstrap.cc:492] T d913e857eb814ccb95455fbc94272b1f P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:22.615267 9703 ts_tablet_manager.cc:1398] T d913e857eb814ccb95455fbc94272b1f P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:22.616135 9702 tablet_bootstrap.cc:654] T d913e857eb814ccb95455fbc94272b1f P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:22.617285 9702 tablet_bootstrap.cc:492] T d913e857eb814ccb95455fbc94272b1f P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:22.617369 9702 ts_tablet_manager.cc:1367] T d913e857eb814ccb95455fbc94272b1f P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:22.617759 9702 raft_consensus.cc:348] T d913e857eb814ccb95455fbc94272b1f P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.617843 9702 raft_consensus.cc:374] T d913e857eb814ccb95455fbc94272b1f P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:22.617915 9702 raft_consensus.cc:729] T d913e857eb814ccb95455fbc94272b1f P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:22.618057 9702 consensus_queue.cc:260] T d913e857eb814ccb95455fbc94272b1f P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.618322 9702 ts_tablet_manager.cc:1398] T d913e857eb814ccb95455fbc94272b1f P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:22.642925 9625 raft_consensus.cc:482] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:22.643067 9625 raft_consensus.cc:504] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:22.643548 9625 leader_election.cc:288] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:22.643929 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e7202e760b1b41d6bdbbdea433baf1d3" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" is_pre_election: true
I20240502 01:58:22.644105 8912 raft_consensus.cc:2454] T e7202e760b1b41d6bdbbdea433baf1d3 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:22.644270 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e7202e760b1b41d6bdbbdea433baf1d3" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15" is_pre_election: true
I20240502 01:58:22.644418 8573 leader_election.cc:302] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:22.644455 8373 raft_consensus.cc:2454] T e7202e760b1b41d6bdbbdea433baf1d3 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:22.644588 9625 raft_consensus.cc:2781] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:22.644652 9625 raft_consensus.cc:482] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:22.644699 9625 raft_consensus.cc:3037] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:22.645641 9625 raft_consensus.cc:504] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:22.645960 9625 leader_election.cc:288] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Requested vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:22.646236 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e7202e760b1b41d6bdbbdea433baf1d3" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
I20240502 01:58:22.646251 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e7202e760b1b41d6bdbbdea433baf1d3" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15"
I20240502 01:58:22.646335 8373 raft_consensus.cc:3037] T e7202e760b1b41d6bdbbdea433baf1d3 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:22.646344 8912 raft_consensus.cc:3037] T e7202e760b1b41d6bdbbdea433baf1d3 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:22.647143 8373 raft_consensus.cc:2454] T e7202e760b1b41d6bdbbdea433baf1d3 P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:22.647166 8912 raft_consensus.cc:2454] T e7202e760b1b41d6bdbbdea433baf1d3 P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:22.647449 8574 leader_election.cc:302] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:22.647645 9625 raft_consensus.cc:2781] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:22.647859 9625 raft_consensus.cc:686] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 LEADER]: Becoming Leader. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Running, Role: LEADER
I20240502 01:58:22.648002 9625 consensus_queue.cc:237] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } }
I20240502 01:58:22.649689 8093 catalog_manager.cc:5484] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 reported cstate change: term changed from 0 to 1, leader changed from <none> to ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: HEALTHY } } }
I20240502 01:58:22.809556 9582 raft_consensus.cc:482] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:22.809676 9582 raft_consensus.cc:504] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.810065 9582 leader_election.cc:288] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e325acceaed541869d43c1f762530c2e (127.7.221.67:42267), 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775)
I20240502 01:58:22.810441 8507 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "08022c1e2dee4aefaf0e39645048a512" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e" is_pre_election: true
I20240502 01:58:22.810470 8776 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "08022c1e2dee4aefaf0e39645048a512" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54" is_pre_election: true
I20240502 01:58:22.810627 8776 raft_consensus.cc:2454] T 08022c1e2dee4aefaf0e39645048a512 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 0.
I20240502 01:58:22.810627 8507 raft_consensus.cc:2454] T 08022c1e2dee4aefaf0e39645048a512 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 0.
I20240502 01:58:22.810930 8172 leader_election.cc:302] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:22.811149 9582 raft_consensus.cc:2781] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:22.811209 9582 raft_consensus.cc:482] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:22.811241 9582 raft_consensus.cc:3037] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:22.812088 9582 raft_consensus.cc:504] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.812415 9582 leader_election.cc:288] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 election: Requested vote from peers e325acceaed541869d43c1f762530c2e (127.7.221.67:42267), 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775)
I20240502 01:58:22.812747 8507 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "08022c1e2dee4aefaf0e39645048a512" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e"
I20240502 01:58:22.812747 8776 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "08022c1e2dee4aefaf0e39645048a512" candidate_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54"
I20240502 01:58:22.812882 8507 raft_consensus.cc:3037] T 08022c1e2dee4aefaf0e39645048a512 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:22.812894 8776 raft_consensus.cc:3037] T 08022c1e2dee4aefaf0e39645048a512 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:22.813861 8776 raft_consensus.cc:2454] T 08022c1e2dee4aefaf0e39645048a512 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 1.
I20240502 01:58:22.813861 8507 raft_consensus.cc:2454] T 08022c1e2dee4aefaf0e39645048a512 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e7a8991bb9ff47489bf8ccbaf5ca7fa9 in term 1.
I20240502 01:58:22.814157 8173 leader_election.cc:302] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e325acceaed541869d43c1f762530c2e, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:22.814383 9582 raft_consensus.cc:2781] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:22.814574 9582 raft_consensus.cc:686] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 LEADER]: Becoming Leader. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Running, Role: LEADER
I20240502 01:58:22.814742 9582 consensus_queue.cc:237] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.817091 8093 catalog_manager.cc:5484] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 reported cstate change: term changed from 0 to 1, leader changed from <none> to e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:22.916042 9676 raft_consensus.cc:482] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:22.916173 9676 raft_consensus.cc:504] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.916533 9676 leader_election.cc:288] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), e325acceaed541869d43c1f762530c2e (127.7.221.67:42267)
I20240502 01:58:22.916862 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d913e857eb814ccb95455fbc94272b1f" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" is_pre_election: true
I20240502 01:58:22.916929 8507 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d913e857eb814ccb95455fbc94272b1f" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e" is_pre_election: true
I20240502 01:58:22.917021 8240 raft_consensus.cc:2454] T d913e857eb814ccb95455fbc94272b1f P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 0.
I20240502 01:58:22.917095 8507 raft_consensus.cc:2454] T d913e857eb814ccb95455fbc94272b1f P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 0.
I20240502 01:58:22.917371 8709 leader_election.cc:302] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e325acceaed541869d43c1f762530c2e; no voters:
I20240502 01:58:22.917608 9676 raft_consensus.cc:2781] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:22.917672 9676 raft_consensus.cc:482] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:22.917719 9676 raft_consensus.cc:3037] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:22.918557 9676 raft_consensus.cc:504] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.918890 9676 leader_election.cc:288] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 election: Requested vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), e325acceaed541869d43c1f762530c2e (127.7.221.67:42267)
I20240502 01:58:22.919183 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d913e857eb814ccb95455fbc94272b1f" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
I20240502 01:58:22.919245 8507 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d913e857eb814ccb95455fbc94272b1f" candidate_uuid: "32416365804d40db9d644b5d32301d54" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e325acceaed541869d43c1f762530c2e"
I20240502 01:58:22.919292 8240 raft_consensus.cc:3037] T d913e857eb814ccb95455fbc94272b1f P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:22.919358 8507 raft_consensus.cc:3037] T d913e857eb814ccb95455fbc94272b1f P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:22.920308 8240 raft_consensus.cc:2454] T d913e857eb814ccb95455fbc94272b1f P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 1.
I20240502 01:58:22.920320 8507 raft_consensus.cc:2454] T d913e857eb814ccb95455fbc94272b1f P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32416365804d40db9d644b5d32301d54 in term 1.
I20240502 01:58:22.920609 8710 leader_election.cc:302] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 32416365804d40db9d644b5d32301d54, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:22.920781 9676 raft_consensus.cc:2781] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:22.920882 9676 raft_consensus.cc:686] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [term 1 LEADER]: Becoming Leader. State: Replica: 32416365804d40db9d644b5d32301d54, State: Running, Role: LEADER
I20240502 01:58:22.921046 9676 consensus_queue.cc:237] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } }
I20240502 01:58:22.924041 8093 catalog_manager.cc:5484] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 reported cstate change: term changed from 0 to 1, leader changed from <none> to 32416365804d40db9d644b5d32301d54 (127.7.221.69). New cstate: current_term: 1 leader_uuid: "32416365804d40db9d644b5d32301d54" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: HEALTHY } } }
W20240502 01:58:23.077160 9722 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:23.077363 9722 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:23.080765 9722 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_10'
I20240502 01:58:23.123406 9625 consensus_queue.cc:1035] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:23.129021 9717 consensus_queue.cc:1035] T e7202e760b1b41d6bdbbdea433baf1d3 P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:23.165597 8240 raft_consensus.cc:1261] T d913e857eb814ccb95455fbc94272b1f P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Refusing update from remote peer 32416365804d40db9d644b5d32301d54: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:23.166013 9589 consensus_queue.cc:1035] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:23.166086 8507 raft_consensus.cc:1261] T d913e857eb814ccb95455fbc94272b1f P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Refusing update from remote peer 32416365804d40db9d644b5d32301d54: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:23.166558 9589 consensus_queue.cc:1035] T d913e857eb814ccb95455fbc94272b1f P 32416365804d40db9d644b5d32301d54 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:23.174456 8506 raft_consensus.cc:1261] T 08022c1e2dee4aefaf0e39645048a512 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Refusing update from remote peer e7a8991bb9ff47489bf8ccbaf5ca7fa9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:23.174692 8776 raft_consensus.cc:1261] T 08022c1e2dee4aefaf0e39645048a512 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Refusing update from remote peer e7a8991bb9ff47489bf8ccbaf5ca7fa9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:23.174890 9719 consensus_queue.cc:1035] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:23.178236 9719 consensus_queue.cc:1035] T 08022c1e2dee4aefaf0e39645048a512 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 110.447 ms
time per row: 0.0552234 ms
I20240502 01:58:23.234730 8094 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34536:
name: "rebalance_test_table_11"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:23.240585 8620 tablet_service.cc:1467] Processing CreateTablet for tablet 373fd7cd211c443aa4dca7a0cbd5129a (DEFAULT_TABLE table=rebalance_test_table_11 [id=e6c54f013d534ee3bd1316e8735ba4e5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:23.240828 8352 tablet_service.cc:1467] Processing CreateTablet for tablet 680d1f83d02f4232a59026455daa3737 (DEFAULT_TABLE table=rebalance_test_table_11 [id=e6c54f013d534ee3bd1316e8735ba4e5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:23.240929 8620 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 373fd7cd211c443aa4dca7a0cbd5129a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:23.241082 8352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 680d1f83d02f4232a59026455daa3737. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:23.241714 8621 tablet_service.cc:1467] Processing CreateTablet for tablet 680d1f83d02f4232a59026455daa3737 (DEFAULT_TABLE table=rebalance_test_table_11 [id=e6c54f013d534ee3bd1316e8735ba4e5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:23.241952 8621 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 680d1f83d02f4232a59026455daa3737. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:23.242017 8891 tablet_service.cc:1467] Processing CreateTablet for tablet 680d1f83d02f4232a59026455daa3737 (DEFAULT_TABLE table=rebalance_test_table_11 [id=e6c54f013d534ee3bd1316e8735ba4e5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:23.242302 8891 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 680d1f83d02f4232a59026455daa3737. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:23.242930 8351 tablet_service.cc:1467] Processing CreateTablet for tablet 373fd7cd211c443aa4dca7a0cbd5129a (DEFAULT_TABLE table=rebalance_test_table_11 [id=e6c54f013d534ee3bd1316e8735ba4e5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:23.243158 8351 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 373fd7cd211c443aa4dca7a0cbd5129a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:23.243566 8219 tablet_service.cc:1467] Processing CreateTablet for tablet e8c16ba856664dbe8bb82fc13b1a40a6 (DEFAULT_TABLE table=rebalance_test_table_11 [id=e6c54f013d534ee3bd1316e8735ba4e5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:23.243824 8219 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e8c16ba856664dbe8bb82fc13b1a40a6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:23.244622 8757 tablet_service.cc:1467] Processing CreateTablet for tablet e8c16ba856664dbe8bb82fc13b1a40a6 (DEFAULT_TABLE table=rebalance_test_table_11 [id=e6c54f013d534ee3bd1316e8735ba4e5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:23.244913 8757 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e8c16ba856664dbe8bb82fc13b1a40a6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:23.245234 8892 tablet_service.cc:1467] Processing CreateTablet for tablet 373fd7cd211c443aa4dca7a0cbd5129a (DEFAULT_TABLE table=rebalance_test_table_11 [id=e6c54f013d534ee3bd1316e8735ba4e5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:23.245360 9792 tablet_bootstrap.cc:492] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:23.245455 8892 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 373fd7cd211c443aa4dca7a0cbd5129a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:23.246711 9792 tablet_bootstrap.cc:654] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:23.247685 9794 tablet_bootstrap.cc:492] T e8c16ba856664dbe8bb82fc13b1a40a6 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Bootstrap starting.
I20240502 01:58:23.247900 9791 tablet_bootstrap.cc:492] T 680d1f83d02f4232a59026455daa3737 P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:23.248039 8485 tablet_service.cc:1467] Processing CreateTablet for tablet e8c16ba856664dbe8bb82fc13b1a40a6 (DEFAULT_TABLE table=rebalance_test_table_11 [id=e6c54f013d534ee3bd1316e8735ba4e5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:23.248350 8485 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e8c16ba856664dbe8bb82fc13b1a40a6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:23.248699 9791 tablet_bootstrap.cc:654] T 680d1f83d02f4232a59026455daa3737 P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:23.249030 9794 tablet_bootstrap.cc:654] T e8c16ba856664dbe8bb82fc13b1a40a6 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:23.250154 9792 tablet_bootstrap.cc:492] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:23.250239 9792 ts_tablet_manager.cc:1367] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 01:58:23.252162 9792 raft_consensus.cc:348] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:23.252388 9792 raft_consensus.cc:374] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:23.252487 9792 raft_consensus.cc:729] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:23.252761 9798 tablet_bootstrap.cc:492] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e: Bootstrap starting.
I20240502 01:58:23.252972 9792 consensus_queue.cc:260] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:23.253481 9792 ts_tablet_manager.cc:1398] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:23.253715 9798 tablet_bootstrap.cc:654] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:23.253715 9792 tablet_bootstrap.cc:492] T 373fd7cd211c443aa4dca7a0cbd5129a P ef9a99ad58b84fd2ada7f035f27361b1: Bootstrap starting.
I20240502 01:58:23.254783 9792 tablet_bootstrap.cc:654] T 373fd7cd211c443aa4dca7a0cbd5129a P ef9a99ad58b84fd2ada7f035f27361b1: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:23.255795 9794 tablet_bootstrap.cc:492] T e8c16ba856664dbe8bb82fc13b1a40a6 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: No bootstrap required, opened a new log
I20240502 01:58:23.256333 9792 tablet_bootstrap.cc:492] T 373fd7cd211c443aa4dca7a0cbd5129a P ef9a99ad58b84fd2ada7f035f27361b1: No bootstrap required, opened a new log
I20240502 01:58:23.256455 9792 ts_tablet_manager.cc:1367] T 373fd7cd211c443aa4dca7a0cbd5129a P ef9a99ad58b84fd2ada7f035f27361b1: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:23.256971 9792 raft_consensus.cc:348] T 373fd7cd211c443aa4dca7a0cbd5129a P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:23.257159 9792 raft_consensus.cc:374] T 373fd7cd211c443aa4dca7a0cbd5129a P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:23.257254 9792 raft_consensus.cc:729] T 373fd7cd211c443aa4dca7a0cbd5129a P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Initialized, Role: FOLLOWER
I20240502 01:58:23.257512 9792 consensus_queue.cc:260] T 373fd7cd211c443aa4dca7a0cbd5129a P ef9a99ad58b84fd2ada7f035f27361b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:23.257920 9792 ts_tablet_manager.cc:1398] T 373fd7cd211c443aa4dca7a0cbd5129a P ef9a99ad58b84fd2ada7f035f27361b1: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:23.258163 9794 ts_tablet_manager.cc:1367] T e8c16ba856664dbe8bb82fc13b1a40a6 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.000s
I20240502 01:58:23.258808 9794 raft_consensus.cc:348] T e8c16ba856664dbe8bb82fc13b1a40a6 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:23.258972 9794 raft_consensus.cc:374] T e8c16ba856664dbe8bb82fc13b1a40a6 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:23.259080 9794 raft_consensus.cc:729] T e8c16ba856664dbe8bb82fc13b1a40a6 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e7a8991bb9ff47489bf8ccbaf5ca7fa9, State: Initialized, Role: FOLLOWER
I20240502 01:58:23.259265 9794 consensus_queue.cc:260] T e8c16ba856664dbe8bb82fc13b1a40a6 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:23.260577 9793 tablet_bootstrap.cc:492] T 680d1f83d02f4232a59026455daa3737 P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:23.261224 9794 ts_tablet_manager.cc:1398] T e8c16ba856664dbe8bb82fc13b1a40a6 P e7a8991bb9ff47489bf8ccbaf5ca7fa9: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:23.263082 9793 tablet_bootstrap.cc:654] T 680d1f83d02f4232a59026455daa3737 P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:23.263247 9791 tablet_bootstrap.cc:492] T 680d1f83d02f4232a59026455daa3737 P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:23.263356 9791 ts_tablet_manager.cc:1367] T 680d1f83d02f4232a59026455daa3737 P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.016s user 0.001s sys 0.000s
I20240502 01:58:23.263783 9799 tablet_bootstrap.cc:492] T e8c16ba856664dbe8bb82fc13b1a40a6 P 32416365804d40db9d644b5d32301d54: Bootstrap starting.
I20240502 01:58:23.263777 9791 raft_consensus.cc:348] T 680d1f83d02f4232a59026455daa3737 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:23.263885 9791 raft_consensus.cc:374] T 680d1f83d02f4232a59026455daa3737 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:23.263921 9791 raft_consensus.cc:729] T 680d1f83d02f4232a59026455daa3737 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:23.264060 9791 consensus_queue.cc:260] T 680d1f83d02f4232a59026455daa3737 P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:23.264281 9793 tablet_bootstrap.cc:492] T 680d1f83d02f4232a59026455daa3737 P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:23.264357 9791 ts_tablet_manager.cc:1398] T 680d1f83d02f4232a59026455daa3737 P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:23.264384 9793 ts_tablet_manager.cc:1367] T 680d1f83d02f4232a59026455daa3737 P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:58:23.264529 9791 tablet_bootstrap.cc:492] T 373fd7cd211c443aa4dca7a0cbd5129a P 73ee70f488334b8f8ffb83aef6456e15: Bootstrap starting.
I20240502 01:58:23.265168 9799 tablet_bootstrap.cc:654] T e8c16ba856664dbe8bb82fc13b1a40a6 P 32416365804d40db9d644b5d32301d54: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:23.265388 9791 tablet_bootstrap.cc:654] T 373fd7cd211c443aa4dca7a0cbd5129a P 73ee70f488334b8f8ffb83aef6456e15: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:23.266073 9793 raft_consensus.cc:348] T 680d1f83d02f4232a59026455daa3737 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:23.266254 9793 raft_consensus.cc:374] T 680d1f83d02f4232a59026455daa3737 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:23.266424 9791 tablet_bootstrap.cc:492] T 373fd7cd211c443aa4dca7a0cbd5129a P 73ee70f488334b8f8ffb83aef6456e15: No bootstrap required, opened a new log
I20240502 01:58:23.266508 9791 ts_tablet_manager.cc:1367] T 373fd7cd211c443aa4dca7a0cbd5129a P 73ee70f488334b8f8ffb83aef6456e15: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:23.267026 9791 raft_consensus.cc:348] T 373fd7cd211c443aa4dca7a0cbd5129a P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:23.267139 9791 raft_consensus.cc:374] T 373fd7cd211c443aa4dca7a0cbd5129a P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:23.267175 9791 raft_consensus.cc:729] T 373fd7cd211c443aa4dca7a0cbd5129a P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73ee70f488334b8f8ffb83aef6456e15, State: Initialized, Role: FOLLOWER
I20240502 01:58:23.266337 9793 raft_consensus.cc:729] T 680d1f83d02f4232a59026455daa3737 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:23.267318 9791 consensus_queue.cc:260] T 373fd7cd211c443aa4dca7a0cbd5129a P 73ee70f488334b8f8ffb83aef6456e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:23.267478 9793 consensus_queue.cc:260] T 680d1f83d02f4232a59026455daa3737 P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:23.267673 9791 ts_tablet_manager.cc:1398] T 373fd7cd211c443aa4dca7a0cbd5129a P 73ee70f488334b8f8ffb83aef6456e15: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:23.267964 9793 ts_tablet_manager.cc:1398] T 680d1f83d02f4232a59026455daa3737 P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:23.268133 9793 tablet_bootstrap.cc:492] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb: Bootstrap starting.
I20240502 01:58:23.269260 9799 tablet_bootstrap.cc:492] T e8c16ba856664dbe8bb82fc13b1a40a6 P 32416365804d40db9d644b5d32301d54: No bootstrap required, opened a new log
I20240502 01:58:23.269404 9799 ts_tablet_manager.cc:1367] T e8c16ba856664dbe8bb82fc13b1a40a6 P 32416365804d40db9d644b5d32301d54: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 01:58:23.269912 9799 raft_consensus.cc:348] T e8c16ba856664dbe8bb82fc13b1a40a6 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:23.270099 9799 raft_consensus.cc:374] T e8c16ba856664dbe8bb82fc13b1a40a6 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:23.270190 9799 raft_consensus.cc:729] T e8c16ba856664dbe8bb82fc13b1a40a6 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32416365804d40db9d644b5d32301d54, State: Initialized, Role: FOLLOWER
I20240502 01:58:23.270336 9799 consensus_queue.cc:260] T e8c16ba856664dbe8bb82fc13b1a40a6 P 32416365804d40db9d644b5d32301d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:23.270516 9793 tablet_bootstrap.cc:654] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:23.271569 9799 ts_tablet_manager.cc:1398] T e8c16ba856664dbe8bb82fc13b1a40a6 P 32416365804d40db9d644b5d32301d54: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:23.272945 9798 tablet_bootstrap.cc:492] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e: No bootstrap required, opened a new log
I20240502 01:58:23.273115 9798 ts_tablet_manager.cc:1367] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e: Time spent bootstrapping tablet: real 0.020s user 0.000s sys 0.002s
I20240502 01:58:23.273670 9798 raft_consensus.cc:348] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:23.273833 9798 raft_consensus.cc:374] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:23.273874 9798 raft_consensus.cc:729] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e325acceaed541869d43c1f762530c2e, State: Initialized, Role: FOLLOWER
I20240502 01:58:23.274097 9798 consensus_queue.cc:260] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:23.274535 9798 ts_tablet_manager.cc:1398] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:23.275833 9793 tablet_bootstrap.cc:492] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb: No bootstrap required, opened a new log
I20240502 01:58:23.275920 9793 ts_tablet_manager.cc:1367] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.000s
I20240502 01:58:23.276329 9793 raft_consensus.cc:348] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:23.276437 9793 raft_consensus.cc:374] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:23.276469 9793 raft_consensus.cc:729] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Initialized, Role: FOLLOWER
I20240502 01:58:23.276625 9793 consensus_queue.cc:260] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:23.276978 9793 ts_tablet_manager.cc:1398] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:23.285916 9717 raft_consensus.cc:482] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:23.286049 9717 raft_consensus.cc:504] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:23.286517 9717 leader_election.cc:288] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:23.286844 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "680d1f83d02f4232a59026455daa3737" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" is_pre_election: true
I20240502 01:58:23.286834 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "680d1f83d02f4232a59026455daa3737" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15" is_pre_election: true
I20240502 01:58:23.286958 8912 raft_consensus.cc:2454] T 680d1f83d02f4232a59026455daa3737 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:23.286983 8373 raft_consensus.cc:2454] T 680d1f83d02f4232a59026455daa3737 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 0.
I20240502 01:58:23.287211 8573 leader_election.cc:302] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:23.287441 9717 raft_consensus.cc:2781] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:23.287499 9717 raft_consensus.cc:482] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:23.287535 9717 raft_consensus.cc:3037] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:23.288388 9717 raft_consensus.cc:504] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:23.288654 9717 leader_election.cc:288] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Requested vote from peers 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503), 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70:35887)
I20240502 01:58:23.288981 8912 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "680d1f83d02f4232a59026455daa3737" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1caeef741efa452fa9bb1fd3d47e65eb"
I20240502 01:58:23.289079 8912 raft_consensus.cc:3037] T 680d1f83d02f4232a59026455daa3737 P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:23.289093 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "680d1f83d02f4232a59026455daa3737" candidate_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15"
I20240502 01:58:23.289183 8373 raft_consensus.cc:3037] T 680d1f83d02f4232a59026455daa3737 P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:23.290014 8912 raft_consensus.cc:2454] T 680d1f83d02f4232a59026455daa3737 P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:23.290019 8373 raft_consensus.cc:2454] T 680d1f83d02f4232a59026455daa3737 P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef9a99ad58b84fd2ada7f035f27361b1 in term 1.
I20240502 01:58:23.290297 8574 leader_election.cc:302] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73ee70f488334b8f8ffb83aef6456e15, ef9a99ad58b84fd2ada7f035f27361b1; no voters:
I20240502 01:58:23.290472 9717 raft_consensus.cc:2781] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:23.290575 9717 raft_consensus.cc:686] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 LEADER]: Becoming Leader. State: Replica: ef9a99ad58b84fd2ada7f035f27361b1, State: Running, Role: LEADER
I20240502 01:58:23.290730 9717 consensus_queue.cc:237] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } }
I20240502 01:58:23.295325 8094 catalog_manager.cc:5484] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 reported cstate change: term changed from 0 to 1, leader changed from <none> to ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:23.308718 9693 raft_consensus.cc:482] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:23.308861 9693 raft_consensus.cc:504] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:23.309298 9693 leader_election.cc:288] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775)
I20240502 01:58:23.309690 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8c16ba856664dbe8bb82fc13b1a40a6" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" is_pre_election: true
I20240502 01:58:23.309775 8776 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8c16ba856664dbe8bb82fc13b1a40a6" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54" is_pre_election: true
I20240502 01:58:23.309861 8240 raft_consensus.cc:2454] T e8c16ba856664dbe8bb82fc13b1a40a6 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 0.
I20240502 01:58:23.309934 8776 raft_consensus.cc:2454] T e8c16ba856664dbe8bb82fc13b1a40a6 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 0.
I20240502 01:58:23.310187 8441 leader_election.cc:302] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e325acceaed541869d43c1f762530c2e, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:23.310353 9693 raft_consensus.cc:2781] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:23.310410 9693 raft_consensus.cc:482] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:23.310444 9693 raft_consensus.cc:3037] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:23.311249 9693 raft_consensus.cc:504] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:23.311532 9693 leader_election.cc:288] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 election: Requested vote from peers e7a8991bb9ff47489bf8ccbaf5ca7fa9 (127.7.221.65:41919), 32416365804d40db9d644b5d32301d54 (127.7.221.69:40775)
I20240502 01:58:23.311830 8240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8c16ba856664dbe8bb82fc13b1a40a6" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9"
I20240502 01:58:23.311944 8240 raft_consensus.cc:3037] T e8c16ba856664dbe8bb82fc13b1a40a6 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:23.312372 8776 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8c16ba856664dbe8bb82fc13b1a40a6" candidate_uuid: "e325acceaed541869d43c1f762530c2e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "32416365804d40db9d644b5d32301d54"
I20240502 01:58:23.312480 8776 raft_consensus.cc:3037] T e8c16ba856664dbe8bb82fc13b1a40a6 P 32416365804d40db9d644b5d32301d54 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:23.312981 8240 raft_consensus.cc:2454] T e8c16ba856664dbe8bb82fc13b1a40a6 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 1.
I20240502 01:58:23.313498 8776 raft_consensus.cc:2454] T e8c16ba856664dbe8bb82fc13b1a40a6 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e325acceaed541869d43c1f762530c2e in term 1.
I20240502 01:58:23.313995 8441 leader_election.cc:302] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e325acceaed541869d43c1f762530c2e, e7a8991bb9ff47489bf8ccbaf5ca7fa9; no voters:
I20240502 01:58:23.314401 9693 raft_consensus.cc:2781] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:23.314610 9693 raft_consensus.cc:686] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [term 1 LEADER]: Becoming Leader. State: Replica: e325acceaed541869d43c1f762530c2e, State: Running, Role: LEADER
I20240502 01:58:23.314754 9693 consensus_queue.cc:237] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } }
I20240502 01:58:23.316488 8094 catalog_manager.cc:5484] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e reported cstate change: term changed from 0 to 1, leader changed from <none> to e325acceaed541869d43c1f762530c2e (127.7.221.67). New cstate: current_term: 1 leader_uuid: "e325acceaed541869d43c1f762530c2e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e325acceaed541869d43c1f762530c2e" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42267 } health_report { overall_health: HEALTHY } } }
I20240502 01:58:23.392519 9630 raft_consensus.cc:482] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:23.392649 9630 raft_consensus.cc:504] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:23.393026 9630 leader_election.cc:288] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021), 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503)
I20240502 01:58:23.393378 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "373fd7cd211c443aa4dca7a0cbd5129a" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15" is_pre_election: true
I20240502 01:58:23.393380 8641 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "373fd7cd211c443aa4dca7a0cbd5129a" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" is_pre_election: true
I20240502 01:58:23.393498 8641 raft_consensus.cc:2454] T 373fd7cd211c443aa4dca7a0cbd5129a P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 0.
I20240502 01:58:23.393513 8373 raft_consensus.cc:2454] T 373fd7cd211c443aa4dca7a0cbd5129a P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 0.
I20240502 01:58:23.393796 8845 leader_election.cc:302] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, 73ee70f488334b8f8ffb83aef6456e15; no voters:
I20240502 01:58:23.393965 9630 raft_consensus.cc:2781] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:23.394029 9630 raft_consensus.cc:482] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:23.394061 9630 raft_consensus.cc:3037] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:23.394889 9630 raft_consensus.cc:504] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:23.395176 9630 leader_election.cc:288] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 election: Requested vote from peers ef9a99ad58b84fd2ada7f035f27361b1 (127.7.221.68:38021), 73ee70f488334b8f8ffb83aef6456e15 (127.7.221.66:37503)
I20240502 01:58:23.395498 8641 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "373fd7cd211c443aa4dca7a0cbd5129a" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef9a99ad58b84fd2ada7f035f27361b1"
I20240502 01:58:23.395514 8373 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "373fd7cd211c443aa4dca7a0cbd5129a" candidate_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73ee70f488334b8f8ffb83aef6456e15"
I20240502 01:58:23.395604 8641 raft_consensus.cc:3037] T 373fd7cd211c443aa4dca7a0cbd5129a P ef9a99ad58b84fd2ada7f035f27361b1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:23.395620 8373 raft_consensus.cc:3037] T 373fd7cd211c443aa4dca7a0cbd5129a P 73ee70f488334b8f8ffb83aef6456e15 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:23.396370 8373 raft_consensus.cc:2454] T 373fd7cd211c443aa4dca7a0cbd5129a P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 1.
I20240502 01:58:23.396641 8845 leader_election.cc:302] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1caeef741efa452fa9bb1fd3d47e65eb, 73ee70f488334b8f8ffb83aef6456e15; no voters:
I20240502 01:58:23.396761 8641 raft_consensus.cc:2454] T 373fd7cd211c443aa4dca7a0cbd5129a P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1caeef741efa452fa9bb1fd3d47e65eb in term 1.
I20240502 01:58:23.396835 9630 raft_consensus.cc:2781] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:23.397058 9630 raft_consensus.cc:686] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 LEADER]: Becoming Leader. State: Replica: 1caeef741efa452fa9bb1fd3d47e65eb, State: Running, Role: LEADER
I20240502 01:58:23.397212 9630 consensus_queue.cc:237] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } }
I20240502 01:58:23.398975 8094 catalog_manager.cc:5484] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb reported cstate change: term changed from 0 to 1, leader changed from <none> to 1caeef741efa452fa9bb1fd3d47e65eb (127.7.221.70). New cstate: current_term: 1 leader_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:23.511700 9812 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:23.511934 9812 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:23.515113 9812 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_11'
I20240502 01:58:23.588642 8912 raft_consensus.cc:1261] T 680d1f83d02f4232a59026455daa3737 P 1caeef741efa452fa9bb1fd3d47e65eb [term 1 FOLLOWER]: Refusing update from remote peer ef9a99ad58b84fd2ada7f035f27361b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:23.589210 9717 consensus_queue.cc:1035] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1caeef741efa452fa9bb1fd3d47e65eb" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 35887 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:23.592164 8373 raft_consensus.cc:1261] T 680d1f83d02f4232a59026455daa3737 P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Refusing update from remote peer ef9a99ad58b84fd2ada7f035f27361b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:23.592327 8372 raft_consensus.cc:1261] T 373fd7cd211c443aa4dca7a0cbd5129a P 73ee70f488334b8f8ffb83aef6456e15 [term 1 FOLLOWER]: Refusing update from remote peer 1caeef741efa452fa9bb1fd3d47e65eb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:23.593713 8240 raft_consensus.cc:1261] T e8c16ba856664dbe8bb82fc13b1a40a6 P e7a8991bb9ff47489bf8ccbaf5ca7fa9 [term 1 FOLLOWER]: Refusing update from remote peer e325acceaed541869d43c1f762530c2e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:23.594219 9810 consensus_queue.cc:1035] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [LEADER]: Connected to new peer: Peer: permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:23.595021 8641 raft_consensus.cc:1261] T 373fd7cd211c443aa4dca7a0cbd5129a P ef9a99ad58b84fd2ada7f035f27361b1 [term 1 FOLLOWER]: Refusing update from remote peer 1caeef741efa452fa9bb1fd3d47e65eb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:23.595351 8776 raft_consensus.cc:1261] T e8c16ba856664dbe8bb82fc13b1a40a6 P 32416365804d40db9d644b5d32301d54 [term 1 FOLLOWER]: Refusing update from remote peer e325acceaed541869d43c1f762530c2e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:23.595350 9693 consensus_queue.cc:1035] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [LEADER]: Connected to new peer: Peer: permanent_uuid: "e7a8991bb9ff47489bf8ccbaf5ca7fa9" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 41919 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:23.595691 9625 consensus_queue.cc:1035] T 680d1f83d02f4232a59026455daa3737 P ef9a99ad58b84fd2ada7f035f27361b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "73ee70f488334b8f8ffb83aef6456e15" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 37503 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:23.603210 9630 consensus_queue.cc:1035] T 373fd7cd211c443aa4dca7a0cbd5129a P 1caeef741efa452fa9bb1fd3d47e65eb [LEADER]: Connected to new peer: Peer: permanent_uuid: "ef9a99ad58b84fd2ada7f035f27361b1" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38021 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:23.604465 9746 mvcc.cc:205] Tried to move back new op lower bound from 7023063464290291712 to 7023063463080062976. Current Snapshot: MvccSnapshot[applied={T|T < 7023063464290291712 or (T in {7023063464290291712})}]
I20240502 01:58:23.605949 9693 consensus_queue.cc:1035] T e8c16ba856664dbe8bb82fc13b1a40a6 P e325acceaed541869d43c1f762530c2e [LEADER]: Connected to new peer: Peer: permanent_uuid: "32416365804d40db9d644b5d32301d54" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 40775 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:23.612231 9749 mvcc.cc:205] Tried to move back new op lower bound from 7023063464308412416 to 7023063463516192768. Current Snapshot: MvccSnapshot[applied={T|T < 7023063464308412416}]
INSERT report
rows total: 2000
time total: 110.626 ms
time per row: 0.055313 ms
I20240502 01:58:23.865063 8892 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:23.867038 8757 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:58:23.867915 8485 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 01:58:23.880497 8620 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:58:23.893513 8351 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:23.900166 8219 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:24.004850 8892 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:24.007674 8757 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:58:24.009368 8351 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:24.017675 8219 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:24.029637 8485 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 01:58:24.030280 8620 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:58:24.171808 8892 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:24.173440 8620 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:58:24.174870 8219 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:24.174974 8757 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:58:24.175329 8351 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:24.175395 8485 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 01:58:24.297376 8892 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:24.299687 8351 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:24.300555 8757 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:58:24.311007 8485 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 01:58:24.312201 8620 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:58:24.328322 8219 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:24.426893 8219 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:24.427425 8892 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:24.427690 8485 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 01:58:24.428025 8351 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:24.436733 8757 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:58:24.451941 8620 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:58:24.570123 8892 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:24.570541 8485 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 01:58:24.573484 8757 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:58:24.574455 8351 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:24.576030 8620 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:58:24.582775 8219 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
8671b1c70fa1447987c2e9c020c526d6 | 127.7.221.126:39109 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.7.221.84:35093 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+---------+----------+----------------+-----------------
1caeef741efa452fa9bb1fd3d47e65eb | 127.7.221.70:35887 | HEALTHY | /C | 5 | 0
32416365804d40db9d644b5d32301d54 | 127.7.221.69:40775 | HEALTHY | /C | 7 | 0
73ee70f488334b8f8ffb83aef6456e15 | 127.7.221.66:37503 | HEALTHY | /A | 4 | 0
e325acceaed541869d43c1f762530c2e | 127.7.221.67:42267 | HEALTHY | /B | 6 | 0
e7a8991bb9ff47489bf8ccbaf5ca7fa9 | 127.7.221.65:41919 | HEALTHY | /A | 5 | 0
ef9a99ad58b84fd2ada7f035f27361b1 | 127.7.221.68:38021 | HEALTHY | /B | 9 | 0
Tablet Server Location Summary
Location | Count
----------+---------
/B | 2
/C | 2
/A | 2
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 6 server(s) checked
local_ip_for_outbound_sockets | 127.7.221.65 | experimental | 127.7.221.65:41919
local_ip_for_outbound_sockets | 127.7.221.66 | experimental | 127.7.221.66:37503
local_ip_for_outbound_sockets | 127.7.221.67 | experimental | 127.7.221.67:42267
local_ip_for_outbound_sockets | 127.7.221.68 | experimental | 127.7.221.68:38021
local_ip_for_outbound_sockets | 127.7.221.69 | experimental | 127.7.221.69:40775
local_ip_for_outbound_sockets | 127.7.221.70 | experimental | 127.7.221.70:35887
never_fsync | true | unsafe,advanced | all 6 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 6 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 6 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 6 server(s) checked
server_dump_info_format | pb | hidden | all 6 server(s) checked
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.7.221.65:41919
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.7.221.66:37503
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.7.221.67:42267
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.7.221.68:38021
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.7.221.69:40775
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingBasicTest.Basic.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.7.221.70:35887
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.7.221.84:35093 | all 6 server(s) checked
time_source | builtin | all 6 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 7 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-------------------------+----+---------+---------------+---------+------------+------------------+-------------
rebalance_test_table_0 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_10 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_11 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_4 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_5 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_6 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_7 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_8 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_9 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 18
First Quartile | 18
Median | 18
Third Quartile | 18
Maximum | 18
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 12
Tablets | 36
Replicas | 108
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240502 01:58:24.623713 8053 log_verifier.cc:126] Checking tablet 05cfe76c4e64414d85ff9a48d2cf1beb
I20240502 01:58:24.653007 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 05cfe76c4e64414d85ff9a48d2cf1beb
I20240502 01:58:24.653115 8053 log_verifier.cc:126] Checking tablet 08022c1e2dee4aefaf0e39645048a512
I20240502 01:58:24.677927 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 08022c1e2dee4aefaf0e39645048a512
I20240502 01:58:24.677994 8053 log_verifier.cc:126] Checking tablet 10e9d48d98ca4da286c329bf76028c05
I20240502 01:58:24.703725 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 10e9d48d98ca4da286c329bf76028c05
I20240502 01:58:24.703791 8053 log_verifier.cc:126] Checking tablet 186a2b637e794290a08e62f8c513a5a0
I20240502 01:58:24.731221 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 186a2b637e794290a08e62f8c513a5a0
I20240502 01:58:24.731287 8053 log_verifier.cc:126] Checking tablet 206508468d004d0185a73f678beb2427
I20240502 01:58:24.773597 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 206508468d004d0185a73f678beb2427
I20240502 01:58:24.773675 8053 log_verifier.cc:126] Checking tablet 209389679cb5447291f54dfe286c67ea
I20240502 01:58:24.808620 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 209389679cb5447291f54dfe286c67ea
I20240502 01:58:24.808687 8053 log_verifier.cc:126] Checking tablet 2e289e488ac2434387d74ea429781bdb
I20240502 01:58:24.866393 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 2e289e488ac2434387d74ea429781bdb
I20240502 01:58:24.866461 8053 log_verifier.cc:126] Checking tablet 2ebb407657b94fb5ae587288859e40fa
I20240502 01:58:24.911082 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 2ebb407657b94fb5ae587288859e40fa
I20240502 01:58:24.911151 8053 log_verifier.cc:126] Checking tablet 373fd7cd211c443aa4dca7a0cbd5129a
I20240502 01:58:24.950503 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 373fd7cd211c443aa4dca7a0cbd5129a
I20240502 01:58:24.950578 8053 log_verifier.cc:126] Checking tablet 39d2476979754af48359409e8fa9554e
I20240502 01:58:24.976133 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 39d2476979754af48359409e8fa9554e
I20240502 01:58:24.976205 8053 log_verifier.cc:126] Checking tablet 3ccb6fa977f849e9af942933c56ef809
I20240502 01:58:25.001737 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 3ccb6fa977f849e9af942933c56ef809
I20240502 01:58:25.001816 8053 log_verifier.cc:126] Checking tablet 54e81d5b68364c0494add244f66b0b6a
I20240502 01:58:25.029173 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 54e81d5b68364c0494add244f66b0b6a
I20240502 01:58:25.029239 8053 log_verifier.cc:126] Checking tablet 54f7129ce08748659fe3c3178d427d67
I20240502 01:58:25.055299 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 54f7129ce08748659fe3c3178d427d67
I20240502 01:58:25.055369 8053 log_verifier.cc:126] Checking tablet 58f576b66ff94cf988d96c2be32232aa
I20240502 01:58:25.082376 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 58f576b66ff94cf988d96c2be32232aa
I20240502 01:58:25.082453 8053 log_verifier.cc:126] Checking tablet 61ed6dd4469c477c8f6352f9c8b25576
I20240502 01:58:25.109350 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 61ed6dd4469c477c8f6352f9c8b25576
I20240502 01:58:25.109426 8053 log_verifier.cc:126] Checking tablet 680d1f83d02f4232a59026455daa3737
I20240502 01:58:25.140101 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 680d1f83d02f4232a59026455daa3737
I20240502 01:58:25.140169 8053 log_verifier.cc:126] Checking tablet 7e5754b0717940b685d5b3330598b7b0
I20240502 01:58:25.166374 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 7e5754b0717940b685d5b3330598b7b0
I20240502 01:58:25.166455 8053 log_verifier.cc:126] Checking tablet 851d391e5df146f9858f5d07bf48a81b
I20240502 01:58:25.194075 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 851d391e5df146f9858f5d07bf48a81b
I20240502 01:58:25.194146 8053 log_verifier.cc:126] Checking tablet 92f03a0caead4668969af4eefe0b6be3
I20240502 01:58:25.221629 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 92f03a0caead4668969af4eefe0b6be3
I20240502 01:58:25.221696 8053 log_verifier.cc:126] Checking tablet 9885bd347924484186d87ba4e4c4ec2e
I20240502 01:58:25.250267 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 9885bd347924484186d87ba4e4c4ec2e
I20240502 01:58:25.250348 8053 log_verifier.cc:126] Checking tablet 99253785df6242b3bc13ca644998b7f9
I20240502 01:58:25.284785 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 99253785df6242b3bc13ca644998b7f9
I20240502 01:58:25.284848 8053 log_verifier.cc:126] Checking tablet 9f467a9793204cedb6c74004423d8e80
I20240502 01:58:25.311420 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 9f467a9793204cedb6c74004423d8e80
I20240502 01:58:25.311493 8053 log_verifier.cc:126] Checking tablet a776c692c47349c7bd02f016b14c5d0f
I20240502 01:58:25.338336 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet a776c692c47349c7bd02f016b14c5d0f
I20240502 01:58:25.338409 8053 log_verifier.cc:126] Checking tablet b1d2473ff348402999cea0f566f31d7f
I20240502 01:58:25.365096 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet b1d2473ff348402999cea0f566f31d7f
I20240502 01:58:25.365166 8053 log_verifier.cc:126] Checking tablet b4ef0bb19ba24ebbaa905a2c5add37ba
I20240502 01:58:25.397719 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet b4ef0bb19ba24ebbaa905a2c5add37ba
I20240502 01:58:25.397786 8053 log_verifier.cc:126] Checking tablet b90c8cce99b94418bf2a4b511cf38e49
I20240502 01:58:25.428637 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet b90c8cce99b94418bf2a4b511cf38e49
I20240502 01:58:25.428699 8053 log_verifier.cc:126] Checking tablet bd1053bf4d774faab67bd6a7fdf03eb0
I20240502 01:58:25.460636 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet bd1053bf4d774faab67bd6a7fdf03eb0
I20240502 01:58:25.460700 8053 log_verifier.cc:126] Checking tablet c49a55dc4dbf48da905d7cfa0bf2fb2a
I20240502 01:58:25.503926 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet c49a55dc4dbf48da905d7cfa0bf2fb2a
I20240502 01:58:25.504009 8053 log_verifier.cc:126] Checking tablet d913e857eb814ccb95455fbc94272b1f
I20240502 01:58:25.555294 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet d913e857eb814ccb95455fbc94272b1f
I20240502 01:58:25.555356 8053 log_verifier.cc:126] Checking tablet e7202e760b1b41d6bdbbdea433baf1d3
I20240502 01:58:25.592005 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet e7202e760b1b41d6bdbbdea433baf1d3
I20240502 01:58:25.592084 8053 log_verifier.cc:126] Checking tablet e8c16ba856664dbe8bb82fc13b1a40a6
I20240502 01:58:25.627305 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet e8c16ba856664dbe8bb82fc13b1a40a6
I20240502 01:58:25.627367 8053 log_verifier.cc:126] Checking tablet ed11274a6a2e4c07b272fe104533ea62
I20240502 01:58:25.656553 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet ed11274a6a2e4c07b272fe104533ea62
I20240502 01:58:25.656620 8053 log_verifier.cc:126] Checking tablet ede42eef861f40419854173443725c83
I20240502 01:58:25.689468 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet ede42eef861f40419854173443725c83
I20240502 01:58:25.689533 8053 log_verifier.cc:126] Checking tablet fb34f32eaf31480abdf7145d76ed76cf
I20240502 01:58:25.724596 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet fb34f32eaf31480abdf7145d76ed76cf
I20240502 01:58:25.724668 8053 log_verifier.cc:126] Checking tablet fc088bb34c27498185a161f2bae4ab3c
I20240502 01:58:25.759214 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet fc088bb34c27498185a161f2bae4ab3c
I20240502 01:58:25.759285 8053 log_verifier.cc:126] Checking tablet ffe9b58784cd4a60993e632ebd3bd32f
I20240502 01:58:25.802390 8053 log_verifier.cc:177] Verified matching terms for 3 ops in tablet ffe9b58784cd4a60993e632ebd3bd32f
I20240502 01:58:25.815469 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 8157
I20240502 01:58:25.823079 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 8289
W20240502 01:58:25.830778 8441 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.65:41919: connect: Connection refused (error 111)
I20240502 01:58:25.830879 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 8422
I20240502 01:58:25.837940 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 8558
W20240502 01:58:25.842449 8709 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.67:42267: connect: Connection refused (error 111)
W20240502 01:58:25.843515 8709 consensus_peers.cc:485] T 2e289e488ac2434387d74ea429781bdb P 32416365804d40db9d644b5d32301d54 -> Peer e325acceaed541869d43c1f762530c2e (127.7.221.67:42267): Couldn't send request to peer e325acceaed541869d43c1f762530c2e. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.67:42267: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:25.844609 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 8692
I20240502 01:58:25.851445 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 8828
I20240502 01:58:25.858142 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 8065
2024-05-02T01:58:25Z chronyd exiting
[ OK ] LocationAwareRebalancingBasicTest.Basic (13920 ms)
[----------] 1 test from LocationAwareRebalancingBasicTest (13920 ms total)
[----------] 1 test from RebalanceStartCriteriaTest
[ RUN ] RebalanceStartCriteriaTest.UnknownIgnoredTServer/1
I20240502 01:58:25.977173 8053 test_util.cc:268] Using random seed: -263095473
I20240502 01:58:25.977870 8053 ts_itest-base.cc:112] Starting cluster with:
I20240502 01:58:25.977906 8053 ts_itest-base.cc:113] --------------
I20240502 01:58:25.977916 8053 ts_itest-base.cc:114] 5 tablet servers
I20240502 01:58:25.977926 8053 ts_itest-base.cc:115] 3 replicas per TS
I20240502 01:58:25.977934 8053 ts_itest-base.cc:116] --------------
2024-05-02T01:58:25Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T01:58:25Z Disabled control of system clock
I20240502 01:58:25.986585 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.7.221.126:45211
--webserver_interface=127.7.221.126
--webserver_port=0
--builtin_ntp_servers=127.7.221.84:46453
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.7.221.126:45211
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false with env {}
W20240502 01:58:26.088605 10145 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:26.088826 10145 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:26.088891 10145 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:26.091830 10145 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:26.091897 10145 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 01:58:26.091929 10145 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:26.091980 10145 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 01:58:26.092043 10145 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 01:58:26.095554 10145 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:46453
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.7.221.126:45211
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.7.221.126:45211
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.7.221.126
--webserver_port=0
--never_fsync=true
--heap_profile_path=/tmp/kudu.10145
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:26.096414 10145 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:26.097452 10145 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:26.104612 10151 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:26.104818 10153 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:26.105542 10150 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:26.106225 10145 server_base.cc:1034] running on GCE node
I20240502 01:58:26.106731 10145 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:26.107625 10145 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:26.108772 10145 hybrid_clock.cc:648] HybridClock initialized: now 1714615106108760 us; error 26 us; skew 500 ppm
I20240502 01:58:26.110673 10145 webserver.cc:460] Webserver started at http://127.7.221.126:41273/ using document root <none> and password file <none>
I20240502 01:58:26.111227 10145 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:26.111284 10145 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:26.111459 10145 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:26.113148 10145 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "9ba918d62dde45c0a0b60a4848f31df9"
format_stamp: "Formatted at 2024-05-02 01:58:26 on dist-test-slave-dmxt"
I20240502 01:58:26.113613 10145 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "9ba918d62dde45c0a0b60a4848f31df9"
format_stamp: "Formatted at 2024-05-02 01:58:26 on dist-test-slave-dmxt"
I20240502 01:58:26.116835 10145 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 01:58:26.118917 10159 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:26.119913 10145 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:26.120026 10145 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
uuid: "9ba918d62dde45c0a0b60a4848f31df9"
format_stamp: "Formatted at 2024-05-02 01:58:26 on dist-test-slave-dmxt"
I20240502 01:58:26.120127 10145 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:26.134097 10145 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:26.134733 10145 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:26.134898 10145 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:26.141115 10145 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.126:45211
I20240502 01:58:26.141136 10212 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.126:45211 every 8 connection(s)
I20240502 01:58:26.142046 10145 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 01:58:26.144570 10213 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:26.149421 10213 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9: Bootstrap starting.
I20240502 01:58:26.151712 10213 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:26.151929 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 10145
I20240502 01:58:26.152069 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 01:58:26.152585 10213 log.cc:824] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:26.154381 10213 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9: No bootstrap required, opened a new log
I20240502 01:58:26.157431 10213 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9ba918d62dde45c0a0b60a4848f31df9" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 45211 } }
I20240502 01:58:26.157652 10213 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:26.157780 10213 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9ba918d62dde45c0a0b60a4848f31df9, State: Initialized, Role: FOLLOWER
I20240502 01:58:26.158227 10213 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9ba918d62dde45c0a0b60a4848f31df9" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 45211 } }
I20240502 01:58:26.158404 10213 raft_consensus.cc:388] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 01:58:26.158474 10213 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 01:58:26.158581 10213 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:26.159432 10213 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9ba918d62dde45c0a0b60a4848f31df9" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 45211 } }
I20240502 01:58:26.159735 10213 leader_election.cc:302] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 9ba918d62dde45c0a0b60a4848f31df9; no voters:
I20240502 01:58:26.159997 10213 leader_election.cc:288] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 01:58:26.160096 10219 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:26.160317 10219 raft_consensus.cc:686] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [term 1 LEADER]: Becoming Leader. State: Replica: 9ba918d62dde45c0a0b60a4848f31df9, State: Running, Role: LEADER
I20240502 01:58:26.160648 10219 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9ba918d62dde45c0a0b60a4848f31df9" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 45211 } }
I20240502 01:58:26.161055 10213 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [sys.catalog]: configured and running, proceeding with master startup.
I20240502 01:58:26.161751 10220 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9ba918d62dde45c0a0b60a4848f31df9. Latest consensus state: current_term: 1 leader_uuid: "9ba918d62dde45c0a0b60a4848f31df9" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9ba918d62dde45c0a0b60a4848f31df9" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 45211 } } }
I20240502 01:58:26.161855 10220 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [sys.catalog]: This master's current role is: LEADER
I20240502 01:58:26.162185 10221 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "9ba918d62dde45c0a0b60a4848f31df9" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9ba918d62dde45c0a0b60a4848f31df9" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 45211 } } }
I20240502 01:58:26.162339 10221 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9 [sys.catalog]: This master's current role is: LEADER
I20240502 01:58:26.162650 10223 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 01:58:26.165149 10223 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 01:58:26.169868 10223 catalog_manager.cc:1347] Generated new cluster ID: 46290fad20aa4c8a851b484abd320457
I20240502 01:58:26.169945 10223 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 01:58:26.191726 10223 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 01:58:26.192713 10223 catalog_manager.cc:1504] Loading token signing keys...
I20240502 01:58:26.200075 10223 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 9ba918d62dde45c0a0b60a4848f31df9: Generated new TSK 0
I20240502 01:58:26.200758 10223 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 01:58:26.205083 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.65:0
--local_ip_for_outbound_sockets=127.7.221.65
--webserver_interface=127.7.221.65
--webserver_port=0
--tserver_master_addrs=127.7.221.126:45211
--builtin_ntp_servers=127.7.221.84:46453
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 01:58:26.314862 10238 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:26.315083 10238 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:26.315133 10238 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:26.318050 10238 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:26.318111 10238 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:26.318181 10238 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.65
I20240502 01:58:26.321560 10238 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:46453
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.7.221.65
--webserver_port=0
--tserver_master_addrs=127.7.221.126:45211
--never_fsync=true
--heap_profile_path=/tmp/kudu.10238
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.65
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:26.322460 10238 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:26.323673 10238 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:26.331873 10246 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:26.331964 10243 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:26.332352 10244 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:26.332903 10238 server_base.cc:1034] running on GCE node
I20240502 01:58:26.333271 10238 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:26.333818 10238 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:26.334959 10238 hybrid_clock.cc:648] HybridClock initialized: now 1714615106334947 us; error 23 us; skew 500 ppm
I20240502 01:58:26.336810 10238 webserver.cc:460] Webserver started at http://127.7.221.65:43591/ using document root <none> and password file <none>
I20240502 01:58:26.337347 10238 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:26.337411 10238 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:26.337587 10238 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:26.339313 10238 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "fba4a2b54a6344e196dcf47556121f5d"
format_stamp: "Formatted at 2024-05-02 01:58:26 on dist-test-slave-dmxt"
I20240502 01:58:26.339774 10238 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "fba4a2b54a6344e196dcf47556121f5d"
format_stamp: "Formatted at 2024-05-02 01:58:26 on dist-test-slave-dmxt"
I20240502 01:58:26.342998 10238 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 01:58:26.345120 10252 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:26.346119 10238 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:26.346225 10238 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "fba4a2b54a6344e196dcf47556121f5d"
format_stamp: "Formatted at 2024-05-02 01:58:26 on dist-test-slave-dmxt"
I20240502 01:58:26.346307 10238 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:26.379559 10238 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:26.380208 10238 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:26.380383 10238 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:26.381017 10238 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:26.381914 10238 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:26.381979 10238 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:26.382041 10238 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:26.382067 10238 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:26.390342 10238 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.65:44523
I20240502 01:58:26.390367 10367 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.65:44523 every 8 connection(s)
I20240502 01:58:26.391302 10238 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 01:58:26.399566 10368 heartbeater.cc:347] Connected to a master server at 127.7.221.126:45211
I20240502 01:58:26.399789 10368 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:26.400306 10368 heartbeater.cc:511] Master 127.7.221.126:45211 requested a full tablet report, sending...
I20240502 01:58:26.401202 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 10238
I20240502 01:58:26.401291 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 01:58:26.401728 10176 ts_manager.cc:194] Registered new tserver with Master: fba4a2b54a6344e196dcf47556121f5d (127.7.221.65:44523)
I20240502 01:58:26.403122 10176 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.65:56747
I20240502 01:58:26.403467 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.66:0
--local_ip_for_outbound_sockets=127.7.221.66
--webserver_interface=127.7.221.66
--webserver_port=0
--tserver_master_addrs=127.7.221.126:45211
--builtin_ntp_servers=127.7.221.84:46453
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 01:58:26.507117 10371 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:26.507320 10371 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:26.507359 10371 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:26.509902 10371 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:26.509951 10371 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:26.510007 10371 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.66
I20240502 01:58:26.513212 10371 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:46453
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.7.221.66
--webserver_port=0
--tserver_master_addrs=127.7.221.126:45211
--never_fsync=true
--heap_profile_path=/tmp/kudu.10371
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.66
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:26.514014 10371 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:26.515029 10371 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:26.523355 10377 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:26.523358 10379 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:26.523618 10376 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:26.524395 10371 server_base.cc:1034] running on GCE node
I20240502 01:58:26.524787 10371 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:26.525344 10371 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:26.526499 10371 hybrid_clock.cc:648] HybridClock initialized: now 1714615106526487 us; error 26 us; skew 500 ppm
I20240502 01:58:26.528354 10371 webserver.cc:460] Webserver started at http://127.7.221.66:37309/ using document root <none> and password file <none>
I20240502 01:58:26.528892 10371 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:26.528947 10371 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:26.529116 10371 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:26.530815 10371 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "59f8c3c10f6f4192a60bee012a66e23f"
format_stamp: "Formatted at 2024-05-02 01:58:26 on dist-test-slave-dmxt"
I20240502 01:58:26.531275 10371 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "59f8c3c10f6f4192a60bee012a66e23f"
format_stamp: "Formatted at 2024-05-02 01:58:26 on dist-test-slave-dmxt"
I20240502 01:58:26.534443 10371 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 01:58:26.536644 10385 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:26.537592 10371 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:26.537706 10371 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "59f8c3c10f6f4192a60bee012a66e23f"
format_stamp: "Formatted at 2024-05-02 01:58:26 on dist-test-slave-dmxt"
I20240502 01:58:26.537787 10371 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:26.550057 10371 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:26.550709 10371 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:26.550879 10371 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:26.551402 10371 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:26.552245 10371 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:26.552296 10371 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:26.552351 10371 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:26.552400 10371 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:26.560473 10371 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.66:40323
I20240502 01:58:26.560509 10499 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.66:40323 every 8 connection(s)
I20240502 01:58:26.561386 10371 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 01:58:26.569182 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 10371
I20240502 01:58:26.569309 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 01:58:26.569629 10500 heartbeater.cc:347] Connected to a master server at 127.7.221.126:45211
I20240502 01:58:26.569859 10500 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:26.570397 10500 heartbeater.cc:511] Master 127.7.221.126:45211 requested a full tablet report, sending...
I20240502 01:58:26.571378 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.67:0
--local_ip_for_outbound_sockets=127.7.221.67
--webserver_interface=127.7.221.67
--webserver_port=0
--tserver_master_addrs=127.7.221.126:45211
--builtin_ntp_servers=127.7.221.84:46453
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:26.571383 10176 ts_manager.cc:194] Registered new tserver with Master: 59f8c3c10f6f4192a60bee012a66e23f (127.7.221.66:40323)
I20240502 01:58:26.571918 10176 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.66:55439
W20240502 01:58:26.672232 10503 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:26.672446 10503 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:26.672488 10503 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:26.675267 10503 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:26.675330 10503 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:26.675385 10503 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.67
I20240502 01:58:26.678546 10503 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:46453
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.7.221.67
--webserver_port=0
--tserver_master_addrs=127.7.221.126:45211
--never_fsync=true
--heap_profile_path=/tmp/kudu.10503
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.67
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:26.679358 10503 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:26.680325 10503 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:26.687847 10508 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:26.687891 10511 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:26.688175 10509 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:26.688889 10503 server_base.cc:1034] running on GCE node
I20240502 01:58:26.689254 10503 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:26.689718 10503 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:26.690863 10503 hybrid_clock.cc:648] HybridClock initialized: now 1714615106690841 us; error 33 us; skew 500 ppm
I20240502 01:58:26.692610 10503 webserver.cc:460] Webserver started at http://127.7.221.67:38695/ using document root <none> and password file <none>
I20240502 01:58:26.693142 10503 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:26.693204 10503 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:26.693372 10503 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:26.694976 10503 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "fcbc3d0583dd4852b94030b353e11d0b"
format_stamp: "Formatted at 2024-05-02 01:58:26 on dist-test-slave-dmxt"
I20240502 01:58:26.695446 10503 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "fcbc3d0583dd4852b94030b353e11d0b"
format_stamp: "Formatted at 2024-05-02 01:58:26 on dist-test-slave-dmxt"
I20240502 01:58:26.698536 10503 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 01:58:26.700664 10517 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:26.701609 10503 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.002s
I20240502 01:58:26.701714 10503 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "fcbc3d0583dd4852b94030b353e11d0b"
format_stamp: "Formatted at 2024-05-02 01:58:26 on dist-test-slave-dmxt"
I20240502 01:58:26.701794 10503 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:26.718421 10503 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:26.719034 10503 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:26.719182 10503 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:26.719671 10503 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:26.720492 10503 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:26.720541 10503 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:26.720600 10503 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:26.720628 10503 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:26.728523 10503 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.67:45781
I20240502 01:58:26.728559 10630 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.67:45781 every 8 connection(s)
I20240502 01:58:26.729385 10503 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 01:58:26.737056 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 10503
I20240502 01:58:26.737052 10631 heartbeater.cc:347] Connected to a master server at 127.7.221.126:45211
I20240502 01:58:26.737139 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 01:58:26.737250 10631 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:26.737699 10631 heartbeater.cc:511] Master 127.7.221.126:45211 requested a full tablet report, sending...
I20240502 01:58:26.738579 10176 ts_manager.cc:194] Registered new tserver with Master: fcbc3d0583dd4852b94030b353e11d0b (127.7.221.67:45781)
I20240502 01:58:26.738986 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.68:0
--local_ip_for_outbound_sockets=127.7.221.68
--webserver_interface=127.7.221.68
--webserver_port=0
--tserver_master_addrs=127.7.221.126:45211
--builtin_ntp_servers=127.7.221.84:46453
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:26.739183 10176 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.67:54683
W20240502 01:58:26.839089 10634 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:26.839282 10634 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:26.839322 10634 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:26.842037 10634 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:26.842128 10634 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:26.842206 10634 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.68
I20240502 01:58:26.845619 10634 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:46453
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.68:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.7.221.68
--webserver_port=0
--tserver_master_addrs=127.7.221.126:45211
--never_fsync=true
--heap_profile_path=/tmp/kudu.10634
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.68
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:26.846419 10634 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:26.847440 10634 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:26.854945 10643 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:26.855127 10641 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:26.855973 10639 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:26.856782 10634 server_base.cc:1034] running on GCE node
I20240502 01:58:26.857147 10634 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:26.857708 10634 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:26.858858 10634 hybrid_clock.cc:648] HybridClock initialized: now 1714615106858849 us; error 29 us; skew 500 ppm
I20240502 01:58:26.860648 10634 webserver.cc:460] Webserver started at http://127.7.221.68:41395/ using document root <none> and password file <none>
I20240502 01:58:26.861164 10634 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:26.861239 10634 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:26.861411 10634 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:26.863138 10634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "6e33e6823caa49dca3a9b32050279a1d"
format_stamp: "Formatted at 2024-05-02 01:58:26 on dist-test-slave-dmxt"
I20240502 01:58:26.863637 10634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "6e33e6823caa49dca3a9b32050279a1d"
format_stamp: "Formatted at 2024-05-02 01:58:26 on dist-test-slave-dmxt"
I20240502 01:58:26.866771 10634 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 01:58:26.868839 10649 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:26.869773 10634 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:26.869879 10634 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "6e33e6823caa49dca3a9b32050279a1d"
format_stamp: "Formatted at 2024-05-02 01:58:26 on dist-test-slave-dmxt"
I20240502 01:58:26.869957 10634 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:26.892820 10634 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:26.893429 10634 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:26.893586 10634 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:26.894114 10634 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:26.895026 10634 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:26.895084 10634 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:26.895143 10634 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:26.895169 10634 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:26.903328 10634 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.68:36273
I20240502 01:58:26.903362 10762 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.68:36273 every 8 connection(s)
I20240502 01:58:26.904225 10634 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 01:58:26.904428 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 10634
I20240502 01:58:26.904523 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 01:58:26.906909 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.69:0
--local_ip_for_outbound_sockets=127.7.221.69
--webserver_interface=127.7.221.69
--webserver_port=0
--tserver_master_addrs=127.7.221.126:45211
--builtin_ntp_servers=127.7.221.84:46453
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:26.913702 10763 heartbeater.cc:347] Connected to a master server at 127.7.221.126:45211
I20240502 01:58:26.913909 10763 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:26.914369 10763 heartbeater.cc:511] Master 127.7.221.126:45211 requested a full tablet report, sending...
I20240502 01:58:26.915370 10176 ts_manager.cc:194] Registered new tserver with Master: 6e33e6823caa49dca3a9b32050279a1d (127.7.221.68:36273)
I20240502 01:58:26.916146 10176 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.68:48101
W20240502 01:58:27.011641 10766 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:27.011847 10766 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:27.011888 10766 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:27.014657 10766 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:27.014707 10766 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:27.014768 10766 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.69
I20240502 01:58:27.017969 10766 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:46453
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.69:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.7.221.69
--webserver_port=0
--tserver_master_addrs=127.7.221.126:45211
--never_fsync=true
--heap_profile_path=/tmp/kudu.10766
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.69
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:27.018729 10766 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:27.019747 10766 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:27.027338 10772 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:27.027526 10771 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:27.028290 10774 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:27.029140 10766 server_base.cc:1034] running on GCE node
I20240502 01:58:27.029479 10766 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:27.029973 10766 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:27.031136 10766 hybrid_clock.cc:648] HybridClock initialized: now 1714615107031123 us; error 22 us; skew 500 ppm
I20240502 01:58:27.033026 10766 webserver.cc:460] Webserver started at http://127.7.221.69:45391/ using document root <none> and password file <none>
I20240502 01:58:27.033548 10766 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:27.033607 10766 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:27.033805 10766 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:27.035411 10766 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "14d3c5c2805a4ff0aa1bbefc533c2495"
format_stamp: "Formatted at 2024-05-02 01:58:27 on dist-test-slave-dmxt"
I20240502 01:58:27.035889 10766 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "14d3c5c2805a4ff0aa1bbefc533c2495"
format_stamp: "Formatted at 2024-05-02 01:58:27 on dist-test-slave-dmxt"
I20240502 01:58:27.039028 10766 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 01:58:27.041153 10780 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:27.042201 10766 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:27.042292 10766 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "14d3c5c2805a4ff0aa1bbefc533c2495"
format_stamp: "Formatted at 2024-05-02 01:58:27 on dist-test-slave-dmxt"
I20240502 01:58:27.042364 10766 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:27.057010 10766 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:27.057554 10766 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:27.057690 10766 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:27.058221 10766 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:27.059123 10766 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:27.059166 10766 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:27.059208 10766 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:27.059237 10766 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:27.067551 10766 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.69:33427
I20240502 01:58:27.067595 10894 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.69:33427 every 8 connection(s)
I20240502 01:58:27.068406 10766 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 01:58:27.073061 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 10766
I20240502 01:58:27.073150 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceStartCriteriaTest.UnknownIgnoredTServer_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 01:58:27.077091 10895 heartbeater.cc:347] Connected to a master server at 127.7.221.126:45211
I20240502 01:58:27.077302 10895 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:27.077842 10895 heartbeater.cc:511] Master 127.7.221.126:45211 requested a full tablet report, sending...
I20240502 01:58:27.078768 10176 ts_manager.cc:194] Registered new tserver with Master: 14d3c5c2805a4ff0aa1bbefc533c2495 (127.7.221.69:33427)
I20240502 01:58:27.079280 10176 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.69:50603
I20240502 01:58:27.087026 8053 external_mini_cluster.cc:930] 5 TS(s) registered with all masters
I20240502 01:58:27.095150 10176 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56992:
name: "TestTable"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
owner: "alice"
I20240502 01:58:27.112607 10829 tablet_service.cc:1467] Processing CreateTablet for tablet 6b44cec5b1b3459fb37c4504e371ac0e (DEFAULT_TABLE table=TestTable [id=73a156a804cf41ac9a46585bbabba5af]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:27.112612 10697 tablet_service.cc:1467] Processing CreateTablet for tablet 6b44cec5b1b3459fb37c4504e371ac0e (DEFAULT_TABLE table=TestTable [id=73a156a804cf41ac9a46585bbabba5af]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:27.112609 10301 tablet_service.cc:1467] Processing CreateTablet for tablet 6b44cec5b1b3459fb37c4504e371ac0e (DEFAULT_TABLE table=TestTable [id=73a156a804cf41ac9a46585bbabba5af]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:27.113495 10829 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6b44cec5b1b3459fb37c4504e371ac0e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:27.113507 10697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6b44cec5b1b3459fb37c4504e371ac0e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:27.113507 10301 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6b44cec5b1b3459fb37c4504e371ac0e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:27.118283 10912 tablet_bootstrap.cc:492] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495: Bootstrap starting.
I20240502 01:58:27.119462 10914 tablet_bootstrap.cc:492] T 6b44cec5b1b3459fb37c4504e371ac0e P 6e33e6823caa49dca3a9b32050279a1d: Bootstrap starting.
I20240502 01:58:27.120445 10912 tablet_bootstrap.cc:654] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:27.121162 10914 tablet_bootstrap.cc:654] T 6b44cec5b1b3459fb37c4504e371ac0e P 6e33e6823caa49dca3a9b32050279a1d: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:27.121217 10912 log.cc:824] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:27.121763 10914 log.cc:824] T 6b44cec5b1b3459fb37c4504e371ac0e P 6e33e6823caa49dca3a9b32050279a1d: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:27.122308 10913 tablet_bootstrap.cc:492] T 6b44cec5b1b3459fb37c4504e371ac0e P fba4a2b54a6344e196dcf47556121f5d: Bootstrap starting.
I20240502 01:58:27.122833 10912 tablet_bootstrap.cc:492] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495: No bootstrap required, opened a new log
I20240502 01:58:27.123019 10912 ts_tablet_manager.cc:1367] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240502 01:58:27.123384 10914 tablet_bootstrap.cc:492] T 6b44cec5b1b3459fb37c4504e371ac0e P 6e33e6823caa49dca3a9b32050279a1d: No bootstrap required, opened a new log
I20240502 01:58:27.123594 10914 ts_tablet_manager.cc:1367] T 6b44cec5b1b3459fb37c4504e371ac0e P 6e33e6823caa49dca3a9b32050279a1d: Time spent bootstrapping tablet: real 0.004s user 0.004s sys 0.000s
I20240502 01:58:27.124289 10913 tablet_bootstrap.cc:654] T 6b44cec5b1b3459fb37c4504e371ac0e P fba4a2b54a6344e196dcf47556121f5d: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:27.125109 10913 log.cc:824] T 6b44cec5b1b3459fb37c4504e371ac0e P fba4a2b54a6344e196dcf47556121f5d: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:27.126276 10912 raft_consensus.cc:348] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14d3c5c2805a4ff0aa1bbefc533c2495" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 33427 } } peers { permanent_uuid: "fba4a2b54a6344e196dcf47556121f5d" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 44523 } } peers { permanent_uuid: "6e33e6823caa49dca3a9b32050279a1d" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36273 } }
I20240502 01:58:27.126606 10912 raft_consensus.cc:374] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:27.126681 10912 raft_consensus.cc:729] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 14d3c5c2805a4ff0aa1bbefc533c2495, State: Initialized, Role: FOLLOWER
I20240502 01:58:27.126755 10913 tablet_bootstrap.cc:492] T 6b44cec5b1b3459fb37c4504e371ac0e P fba4a2b54a6344e196dcf47556121f5d: No bootstrap required, opened a new log
I20240502 01:58:27.126916 10913 ts_tablet_manager.cc:1367] T 6b44cec5b1b3459fb37c4504e371ac0e P fba4a2b54a6344e196dcf47556121f5d: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240502 01:58:27.127120 10912 consensus_queue.cc:260] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14d3c5c2805a4ff0aa1bbefc533c2495" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 33427 } } peers { permanent_uuid: "fba4a2b54a6344e196dcf47556121f5d" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 44523 } } peers { permanent_uuid: "6e33e6823caa49dca3a9b32050279a1d" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36273 } }
I20240502 01:58:27.127323 10914 raft_consensus.cc:348] T 6b44cec5b1b3459fb37c4504e371ac0e P 6e33e6823caa49dca3a9b32050279a1d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14d3c5c2805a4ff0aa1bbefc533c2495" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 33427 } } peers { permanent_uuid: "fba4a2b54a6344e196dcf47556121f5d" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 44523 } } peers { permanent_uuid: "6e33e6823caa49dca3a9b32050279a1d" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36273 } }
I20240502 01:58:27.127568 10914 raft_consensus.cc:374] T 6b44cec5b1b3459fb37c4504e371ac0e P 6e33e6823caa49dca3a9b32050279a1d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:27.127663 10914 raft_consensus.cc:729] T 6b44cec5b1b3459fb37c4504e371ac0e P 6e33e6823caa49dca3a9b32050279a1d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6e33e6823caa49dca3a9b32050279a1d, State: Initialized, Role: FOLLOWER
I20240502 01:58:27.127736 10895 heartbeater.cc:503] Master 127.7.221.126:45211 was elected leader, sending a full tablet report...
I20240502 01:58:27.127961 10912 ts_tablet_manager.cc:1398] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s
I20240502 01:58:27.128046 10914 consensus_queue.cc:260] T 6b44cec5b1b3459fb37c4504e371ac0e P 6e33e6823caa49dca3a9b32050279a1d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14d3c5c2805a4ff0aa1bbefc533c2495" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 33427 } } peers { permanent_uuid: "fba4a2b54a6344e196dcf47556121f5d" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 44523 } } peers { permanent_uuid: "6e33e6823caa49dca3a9b32050279a1d" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36273 } }
I20240502 01:58:27.128801 10914 ts_tablet_manager.cc:1398] T 6b44cec5b1b3459fb37c4504e371ac0e P 6e33e6823caa49dca3a9b32050279a1d: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s
I20240502 01:58:27.128837 10763 heartbeater.cc:503] Master 127.7.221.126:45211 was elected leader, sending a full tablet report...
I20240502 01:58:27.130797 10913 raft_consensus.cc:348] T 6b44cec5b1b3459fb37c4504e371ac0e P fba4a2b54a6344e196dcf47556121f5d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14d3c5c2805a4ff0aa1bbefc533c2495" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 33427 } } peers { permanent_uuid: "fba4a2b54a6344e196dcf47556121f5d" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 44523 } } peers { permanent_uuid: "6e33e6823caa49dca3a9b32050279a1d" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36273 } }
I20240502 01:58:27.130980 10913 raft_consensus.cc:374] T 6b44cec5b1b3459fb37c4504e371ac0e P fba4a2b54a6344e196dcf47556121f5d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:27.131072 10913 raft_consensus.cc:729] T 6b44cec5b1b3459fb37c4504e371ac0e P fba4a2b54a6344e196dcf47556121f5d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fba4a2b54a6344e196dcf47556121f5d, State: Initialized, Role: FOLLOWER
I20240502 01:58:27.131424 10913 consensus_queue.cc:260] T 6b44cec5b1b3459fb37c4504e371ac0e P fba4a2b54a6344e196dcf47556121f5d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14d3c5c2805a4ff0aa1bbefc533c2495" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 33427 } } peers { permanent_uuid: "fba4a2b54a6344e196dcf47556121f5d" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 44523 } } peers { permanent_uuid: "6e33e6823caa49dca3a9b32050279a1d" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36273 } }
I20240502 01:58:27.132050 10368 heartbeater.cc:503] Master 127.7.221.126:45211 was elected leader, sending a full tablet report...
I20240502 01:58:27.132140 10913 ts_tablet_manager.cc:1398] T 6b44cec5b1b3459fb37c4504e371ac0e P fba4a2b54a6344e196dcf47556121f5d: Time spent starting tablet: real 0.005s user 0.004s sys 0.000s
W20240502 01:58:27.142534 10369 tablet.cc:2369] T 6b44cec5b1b3459fb37c4504e371ac0e P fba4a2b54a6344e196dcf47556121f5d: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 01:58:27.155268 10764 tablet.cc:2369] T 6b44cec5b1b3459fb37c4504e371ac0e P 6e33e6823caa49dca3a9b32050279a1d: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 01:58:27.290789 10918 raft_consensus.cc:482] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:27.290972 10918 raft_consensus.cc:504] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14d3c5c2805a4ff0aa1bbefc533c2495" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 33427 } } peers { permanent_uuid: "fba4a2b54a6344e196dcf47556121f5d" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 44523 } } peers { permanent_uuid: "6e33e6823caa49dca3a9b32050279a1d" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36273 } }
I20240502 01:58:27.292076 10918 leader_election.cc:288] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fba4a2b54a6344e196dcf47556121f5d (127.7.221.65:44523), 6e33e6823caa49dca3a9b32050279a1d (127.7.221.68:36273)
I20240502 01:58:27.295594 10717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6b44cec5b1b3459fb37c4504e371ac0e" candidate_uuid: "14d3c5c2805a4ff0aa1bbefc533c2495" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6e33e6823caa49dca3a9b32050279a1d" is_pre_election: true
I20240502 01:58:27.295584 10321 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6b44cec5b1b3459fb37c4504e371ac0e" candidate_uuid: "14d3c5c2805a4ff0aa1bbefc533c2495" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fba4a2b54a6344e196dcf47556121f5d" is_pre_election: true
I20240502 01:58:27.295881 10717 raft_consensus.cc:2454] T 6b44cec5b1b3459fb37c4504e371ac0e P 6e33e6823caa49dca3a9b32050279a1d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 14d3c5c2805a4ff0aa1bbefc533c2495 in term 0.
I20240502 01:58:27.295881 10321 raft_consensus.cc:2454] T 6b44cec5b1b3459fb37c4504e371ac0e P fba4a2b54a6344e196dcf47556121f5d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 14d3c5c2805a4ff0aa1bbefc533c2495 in term 0.
I20240502 01:58:27.296321 10782 leader_election.cc:302] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 14d3c5c2805a4ff0aa1bbefc533c2495, fba4a2b54a6344e196dcf47556121f5d; no voters:
I20240502 01:58:27.296557 10918 raft_consensus.cc:2781] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:27.296628 10918 raft_consensus.cc:482] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:27.296674 10918 raft_consensus.cc:3037] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:27.297519 10918 raft_consensus.cc:504] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14d3c5c2805a4ff0aa1bbefc533c2495" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 33427 } } peers { permanent_uuid: "fba4a2b54a6344e196dcf47556121f5d" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 44523 } } peers { permanent_uuid: "6e33e6823caa49dca3a9b32050279a1d" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36273 } }
I20240502 01:58:27.297827 10918 leader_election.cc:288] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [CANDIDATE]: Term 1 election: Requested vote from peers fba4a2b54a6344e196dcf47556121f5d (127.7.221.65:44523), 6e33e6823caa49dca3a9b32050279a1d (127.7.221.68:36273)
I20240502 01:58:27.298170 10321 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6b44cec5b1b3459fb37c4504e371ac0e" candidate_uuid: "14d3c5c2805a4ff0aa1bbefc533c2495" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fba4a2b54a6344e196dcf47556121f5d"
I20240502 01:58:27.298175 10717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6b44cec5b1b3459fb37c4504e371ac0e" candidate_uuid: "14d3c5c2805a4ff0aa1bbefc533c2495" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6e33e6823caa49dca3a9b32050279a1d"
I20240502 01:58:27.298285 10717 raft_consensus.cc:3037] T 6b44cec5b1b3459fb37c4504e371ac0e P 6e33e6823caa49dca3a9b32050279a1d [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:27.298271 10321 raft_consensus.cc:3037] T 6b44cec5b1b3459fb37c4504e371ac0e P fba4a2b54a6344e196dcf47556121f5d [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:27.299186 10321 raft_consensus.cc:2454] T 6b44cec5b1b3459fb37c4504e371ac0e P fba4a2b54a6344e196dcf47556121f5d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 14d3c5c2805a4ff0aa1bbefc533c2495 in term 1.
I20240502 01:58:27.299203 10717 raft_consensus.cc:2454] T 6b44cec5b1b3459fb37c4504e371ac0e P 6e33e6823caa49dca3a9b32050279a1d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 14d3c5c2805a4ff0aa1bbefc533c2495 in term 1.
I20240502 01:58:27.299464 10782 leader_election.cc:302] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 14d3c5c2805a4ff0aa1bbefc533c2495, fba4a2b54a6344e196dcf47556121f5d; no voters:
I20240502 01:58:27.299638 10918 raft_consensus.cc:2781] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:27.299844 10918 raft_consensus.cc:686] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [term 1 LEADER]: Becoming Leader. State: Replica: 14d3c5c2805a4ff0aa1bbefc533c2495, State: Running, Role: LEADER
I20240502 01:58:27.300148 10918 consensus_queue.cc:237] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14d3c5c2805a4ff0aa1bbefc533c2495" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 33427 } } peers { permanent_uuid: "fba4a2b54a6344e196dcf47556121f5d" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 44523 } } peers { permanent_uuid: "6e33e6823caa49dca3a9b32050279a1d" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36273 } }
I20240502 01:58:27.303205 10176 catalog_manager.cc:5484] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 reported cstate change: term changed from 0 to 1, leader changed from <none> to 14d3c5c2805a4ff0aa1bbefc533c2495 (127.7.221.69). New cstate: current_term: 1 leader_uuid: "14d3c5c2805a4ff0aa1bbefc533c2495" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14d3c5c2805a4ff0aa1bbefc533c2495" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 33427 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fba4a2b54a6344e196dcf47556121f5d" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 44523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6e33e6823caa49dca3a9b32050279a1d" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36273 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:27.319281 10896 tablet.cc:2369] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 01:58:27.339015 8053 external_mini_cluster.cc:930] 5 TS(s) registered with all masters
I20240502 01:58:27.340440 8053 ts_itest-base.cc:233] Waiting for 1 tablets on tserver fba4a2b54a6344e196dcf47556121f5d to finish bootstrapping
I20240502 01:58:27.345552 8053 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 59f8c3c10f6f4192a60bee012a66e23f to finish bootstrapping
I20240502 01:58:27.350661 8053 ts_itest-base.cc:233] Waiting for 0 tablets on tserver fcbc3d0583dd4852b94030b353e11d0b to finish bootstrapping
I20240502 01:58:27.355520 8053 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 6e33e6823caa49dca3a9b32050279a1d to finish bootstrapping
I20240502 01:58:27.359755 8053 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 14d3c5c2805a4ff0aa1bbefc533c2495 to finish bootstrapping
I20240502 01:58:27.534549 10301 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:27.534734 10829 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:27.535713 10697 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:27.536314 10433 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:27.542891 10565 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:27.573938 10500 heartbeater.cc:503] Master 127.7.221.126:45211 was elected leader, sending a full tablet report...
I20240502 01:58:27.623471 10829 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:27.623626 10433 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:27.623932 10697 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:27.625299 10301 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:27.626529 10565 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:27.726341 10918 consensus_queue.cc:1035] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6e33e6823caa49dca3a9b32050279a1d" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36273 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:27.732403 10922 consensus_queue.cc:1035] T 6b44cec5b1b3459fb37c4504e371ac0e P 14d3c5c2805a4ff0aa1bbefc533c2495 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fba4a2b54a6344e196dcf47556121f5d" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 44523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:27.740787 10631 heartbeater.cc:503] Master 127.7.221.126:45211 was elected leader, sending a full tablet report...
I20240502 01:58:27.834887 10565 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:27.835471 10829 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:27.836251 10697 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:27.836781 10301 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:27.837541 10433 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:27.924590 10433 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:27.924655 10697 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:27.925967 10565 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:27.926051 10829 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:27.927301 10301 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:28.026058 10829 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:28.026616 10565 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:28.028075 10433 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:28.028707 10301 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:28.029997 10697 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:28.073192 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 10238
I20240502 01:58:28.079965 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 10371
I20240502 01:58:28.086694 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 10503
I20240502 01:58:28.092764 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 10634
I20240502 01:58:28.099092 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 10766
I20240502 01:58:28.105042 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 10145
2024-05-02T01:58:28Z chronyd exiting
[ OK ] RebalanceStartCriteriaTest.UnknownIgnoredTServer/1 (2150 ms)
[----------] 1 test from RebalanceStartCriteriaTest (2150 ms total)
[----------] 1 test from RebalanceParamTest
[ RUN ] RebalanceParamTest.Rebalance/3
I20240502 01:58:28.127844 8053 test_util.cc:268] Using random seed: -260944801
I20240502 01:58:28.128624 8053 ts_itest-base.cc:112] Starting cluster with:
I20240502 01:58:28.128671 8053 ts_itest-base.cc:113] --------------
I20240502 01:58:28.128683 8053 ts_itest-base.cc:114] 7 tablet servers
I20240502 01:58:28.128695 8053 ts_itest-base.cc:115] 2 replicas per TS
I20240502 01:58:28.128705 8053 ts_itest-base.cc:116] --------------
2024-05-02T01:58:28Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T01:58:28Z Disabled control of system clock
I20240502 01:58:28.136687 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.7.221.126:32951
--webserver_interface=127.7.221.126
--webserver_port=0
--builtin_ntp_servers=127.7.221.84:39105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.7.221.126:32951
--allow_unsafe_replication_factor
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--tserver_unresponsive_timeout_ms=3000 with env {}
W20240502 01:58:28.245466 11111 flags.cc:425] Enabled unsafe flag: --allow_unsafe_replication_factor=true
W20240502 01:58:28.245743 11111 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:28.245806 11111 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:28.245879 11111 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:28.248803 11111 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:28.248885 11111 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 01:58:28.248927 11111 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:28.248965 11111 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 01:58:28.248999 11111 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 01:58:28.252655 11111 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
--allow_unsafe_replication_factor=true
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.7.221.126:32951
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.7.221.126:32951
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.7.221.126
--webserver_port=0
--never_fsync=true
--heap_profile_path=/tmp/kudu.11111
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:28.253568 11111 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:28.254732 11111 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:28.261714 11117 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:28.262316 11119 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:28.262519 11116 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:28.263196 11111 server_base.cc:1034] running on GCE node
I20240502 01:58:28.263682 11111 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:28.264569 11111 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:28.265722 11111 hybrid_clock.cc:648] HybridClock initialized: now 1714615108265704 us; error 31 us; skew 500 ppm
I20240502 01:58:28.267601 11111 webserver.cc:460] Webserver started at http://127.7.221.126:38199/ using document root <none> and password file <none>
I20240502 01:58:28.268136 11111 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:28.268193 11111 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:28.268379 11111 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:28.269977 11111 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "629fef51a73345408bea08d1a5ecc913"
format_stamp: "Formatted at 2024-05-02 01:58:28 on dist-test-slave-dmxt"
I20240502 01:58:28.270452 11111 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "629fef51a73345408bea08d1a5ecc913"
format_stamp: "Formatted at 2024-05-02 01:58:28 on dist-test-slave-dmxt"
I20240502 01:58:28.273691 11111 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.001s sys 0.001s
I20240502 01:58:28.275833 11125 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:28.276742 11111 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:28.276859 11111 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
uuid: "629fef51a73345408bea08d1a5ecc913"
format_stamp: "Formatted at 2024-05-02 01:58:28 on dist-test-slave-dmxt"
I20240502 01:58:28.276939 11111 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:28.294011 11111 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:28.294682 11111 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:28.294836 11111 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:28.301102 11178 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.126:32951 every 8 connection(s)
I20240502 01:58:28.301100 11111 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.126:32951
I20240502 01:58:28.302073 11111 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 01:58:28.304586 11179 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:28.309302 11179 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913: Bootstrap starting.
I20240502 01:58:28.311372 11179 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:28.311902 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 11111
I20240502 01:58:28.312026 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 01:58:28.312122 11179 log.cc:824] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:28.313947 11179 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913: No bootstrap required, opened a new log
I20240502 01:58:28.316293 11179 raft_consensus.cc:348] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "629fef51a73345408bea08d1a5ecc913" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 32951 } }
I20240502 01:58:28.316499 11179 raft_consensus.cc:374] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:28.316577 11179 raft_consensus.cc:729] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 629fef51a73345408bea08d1a5ecc913, State: Initialized, Role: FOLLOWER
I20240502 01:58:28.316969 11179 consensus_queue.cc:260] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "629fef51a73345408bea08d1a5ecc913" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 32951 } }
I20240502 01:58:28.317095 11179 raft_consensus.cc:388] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 01:58:28.317191 11179 raft_consensus.cc:482] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 01:58:28.317340 11179 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:28.318182 11179 raft_consensus.cc:504] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "629fef51a73345408bea08d1a5ecc913" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 32951 } }
I20240502 01:58:28.318511 11179 leader_election.cc:302] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 629fef51a73345408bea08d1a5ecc913; no voters:
I20240502 01:58:28.318817 11179 leader_election.cc:288] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 01:58:28.319204 11184 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:28.319433 11184 raft_consensus.cc:686] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [term 1 LEADER]: Becoming Leader. State: Replica: 629fef51a73345408bea08d1a5ecc913, State: Running, Role: LEADER
I20240502 01:58:28.319867 11179 sys_catalog.cc:564] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [sys.catalog]: configured and running, proceeding with master startup.
I20240502 01:58:28.320540 11184 consensus_queue.cc:237] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "629fef51a73345408bea08d1a5ecc913" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 32951 } }
I20240502 01:58:28.321916 11186 sys_catalog.cc:455] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 629fef51a73345408bea08d1a5ecc913. Latest consensus state: current_term: 1 leader_uuid: "629fef51a73345408bea08d1a5ecc913" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "629fef51a73345408bea08d1a5ecc913" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 32951 } } }
I20240502 01:58:28.321970 11185 sys_catalog.cc:455] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "629fef51a73345408bea08d1a5ecc913" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "629fef51a73345408bea08d1a5ecc913" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 32951 } } }
I20240502 01:58:28.322185 11186 sys_catalog.cc:458] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [sys.catalog]: This master's current role is: LEADER
I20240502 01:58:28.322190 11185 sys_catalog.cc:458] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913 [sys.catalog]: This master's current role is: LEADER
W20240502 01:58:28.324584 11198 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20240502 01:58:28.324644 11198 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20240502 01:58:28.324719 11199 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 01:58:28.325388 11199 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 01:58:28.329986 11199 catalog_manager.cc:1347] Generated new cluster ID: efe1502424264c7ea0a21df0e6c145fa
I20240502 01:58:28.330050 11199 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 01:58:28.347628 11199 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 01:58:28.348492 11199 catalog_manager.cc:1504] Loading token signing keys...
I20240502 01:58:28.356761 11199 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 629fef51a73345408bea08d1a5ecc913: Generated new TSK 0
I20240502 01:58:28.357396 11199 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 01:58:28.371173 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.65:0
--local_ip_for_outbound_sockets=127.7.221.65
--webserver_interface=127.7.221.65
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--builtin_ntp_servers=127.7.221.84:39105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 01:58:28.475914 11203 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:28.476087 11203 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:28.476130 11203 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:28.478796 11203 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:28.478866 11203 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:28.478940 11203 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.65
I20240502 01:58:28.482278 11203 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.7.221.65
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--never_fsync=true
--heap_profile_path=/tmp/kudu.11203
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.65
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:28.483109 11203 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:28.484158 11203 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:28.491310 11209 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:28.491413 11208 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:28.491645 11212 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:28.492466 11203 server_base.cc:1034] running on GCE node
I20240502 01:58:28.492803 11203 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:28.493273 11203 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:28.494415 11203 hybrid_clock.cc:648] HybridClock initialized: now 1714615108494393 us; error 39 us; skew 500 ppm
I20240502 01:58:28.496130 11203 webserver.cc:460] Webserver started at http://127.7.221.65:41915/ using document root <none> and password file <none>
I20240502 01:58:28.496659 11203 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:28.496711 11203 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:28.496892 11203 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:28.498342 11203 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "a47375810d964f128aae1e2df1370b79"
format_stamp: "Formatted at 2024-05-02 01:58:28 on dist-test-slave-dmxt"
I20240502 01:58:28.498808 11203 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "a47375810d964f128aae1e2df1370b79"
format_stamp: "Formatted at 2024-05-02 01:58:28 on dist-test-slave-dmxt"
I20240502 01:58:28.501703 11203 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:28.503784 11218 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:28.504737 11203 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:28.504848 11203 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "a47375810d964f128aae1e2df1370b79"
format_stamp: "Formatted at 2024-05-02 01:58:28 on dist-test-slave-dmxt"
I20240502 01:58:28.504923 11203 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:28.529284 11203 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:28.529915 11203 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:28.530200 11203 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:28.530800 11203 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:28.531735 11203 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:28.531785 11203 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:28.531842 11203 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:28.531874 11203 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:28.540082 11203 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.65:36857
I20240502 01:58:28.540107 11332 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.65:36857 every 8 connection(s)
I20240502 01:58:28.540975 11203 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 01:58:28.547008 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 11203
I20240502 01:58:28.547116 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 01:58:28.549403 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.66:0
--local_ip_for_outbound_sockets=127.7.221.66
--webserver_interface=127.7.221.66
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--builtin_ntp_servers=127.7.221.84:39105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:28.549805 11333 heartbeater.cc:347] Connected to a master server at 127.7.221.126:32951
I20240502 01:58:28.550081 11333 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:28.550735 11333 heartbeater.cc:511] Master 127.7.221.126:32951 requested a full tablet report, sending...
I20240502 01:58:28.552340 11143 ts_manager.cc:194] Registered new tserver with Master: a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857)
I20240502 01:58:28.554049 11143 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.65:49541
W20240502 01:58:28.654083 11336 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:28.654289 11336 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:28.654327 11336 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:28.657116 11336 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:28.657203 11336 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:28.657279 11336 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.66
I20240502 01:58:28.660543 11336 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.7.221.66
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--never_fsync=true
--heap_profile_path=/tmp/kudu.11336
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.66
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:28.661337 11336 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:28.662338 11336 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:28.669677 11342 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:28.669850 11341 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:28.670642 11345 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:28.671370 11336 server_base.cc:1034] running on GCE node
I20240502 01:58:28.671701 11336 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:28.672174 11336 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:28.673322 11336 hybrid_clock.cc:648] HybridClock initialized: now 1714615108673299 us; error 37 us; skew 500 ppm
I20240502 01:58:28.675136 11336 webserver.cc:460] Webserver started at http://127.7.221.66:33729/ using document root <none> and password file <none>
I20240502 01:58:28.675654 11336 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:28.675715 11336 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:28.675894 11336 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:28.677469 11336 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "6bf256be71684cdf87ef861e5890b614"
format_stamp: "Formatted at 2024-05-02 01:58:28 on dist-test-slave-dmxt"
I20240502 01:58:28.677915 11336 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "6bf256be71684cdf87ef861e5890b614"
format_stamp: "Formatted at 2024-05-02 01:58:28 on dist-test-slave-dmxt"
I20240502 01:58:28.681102 11336 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.001s sys 0.004s
I20240502 01:58:28.683241 11351 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:28.684201 11336 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:28.684309 11336 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "6bf256be71684cdf87ef861e5890b614"
format_stamp: "Formatted at 2024-05-02 01:58:28 on dist-test-slave-dmxt"
I20240502 01:58:28.684401 11336 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:28.704341 11336 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:28.705024 11336 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:28.705183 11336 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:28.705722 11336 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:28.706588 11336 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:28.706665 11336 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:28.706722 11336 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:28.706746 11336 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:28.714794 11336 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.66:36035
I20240502 01:58:28.714825 11465 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.66:36035 every 8 connection(s)
I20240502 01:58:28.715633 11336 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 01:58:28.723928 11466 heartbeater.cc:347] Connected to a master server at 127.7.221.126:32951
I20240502 01:58:28.724164 11466 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:28.724663 11466 heartbeater.cc:511] Master 127.7.221.126:32951 requested a full tablet report, sending...
I20240502 01:58:28.725610 11143 ts_manager.cc:194] Registered new tserver with Master: 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:28.725643 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 11336
I20240502 01:58:28.725736 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 01:58:28.726135 11143 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.66:44373
I20240502 01:58:28.727602 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.67:0
--local_ip_for_outbound_sockets=127.7.221.67
--webserver_interface=127.7.221.67
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--builtin_ntp_servers=127.7.221.84:39105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 01:58:28.829962 11469 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:28.830155 11469 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:28.830221 11469 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:28.833007 11469 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:28.833076 11469 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:28.833169 11469 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.67
I20240502 01:58:28.836557 11469 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.7.221.67
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--never_fsync=true
--heap_profile_path=/tmp/kudu.11469
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.67
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:28.837354 11469 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:28.838373 11469 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:28.845666 11474 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:28.845809 11477 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:28.846486 11475 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:28.847285 11469 server_base.cc:1034] running on GCE node
I20240502 01:58:28.847597 11469 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:28.848053 11469 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:28.849189 11469 hybrid_clock.cc:648] HybridClock initialized: now 1714615108849169 us; error 32 us; skew 500 ppm
I20240502 01:58:28.850872 11469 webserver.cc:460] Webserver started at http://127.7.221.67:33065/ using document root <none> and password file <none>
I20240502 01:58:28.851351 11469 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:28.851398 11469 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:28.851536 11469 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:28.853133 11469 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "8c191817e0ce48b3b552af940f9d11ae"
format_stamp: "Formatted at 2024-05-02 01:58:28 on dist-test-slave-dmxt"
I20240502 01:58:28.853561 11469 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "8c191817e0ce48b3b552af940f9d11ae"
format_stamp: "Formatted at 2024-05-02 01:58:28 on dist-test-slave-dmxt"
I20240502 01:58:28.856877 11469 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 01:58:28.859105 11483 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:28.860090 11469 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:28.860188 11469 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "8c191817e0ce48b3b552af940f9d11ae"
format_stamp: "Formatted at 2024-05-02 01:58:28 on dist-test-slave-dmxt"
I20240502 01:58:28.860270 11469 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:28.874889 11469 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:28.875442 11469 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:28.875579 11469 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:28.876046 11469 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:28.876837 11469 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:28.876879 11469 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:28.876921 11469 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:28.876951 11469 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:28.885116 11469 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.67:44991
I20240502 01:58:28.885147 11598 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.67:44991 every 8 connection(s)
I20240502 01:58:28.885999 11469 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 01:58:28.892685 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 11469
I20240502 01:58:28.892817 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 01:58:28.894137 11599 heartbeater.cc:347] Connected to a master server at 127.7.221.126:32951
I20240502 01:58:28.894361 11599 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:28.894889 11599 heartbeater.cc:511] Master 127.7.221.126:32951 requested a full tablet report, sending...
I20240502 01:58:28.895046 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.68:0
--local_ip_for_outbound_sockets=127.7.221.68
--webserver_interface=127.7.221.68
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--builtin_ntp_servers=127.7.221.84:39105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:28.895843 11142 ts_manager.cc:194] Registered new tserver with Master: 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:28.896582 11142 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.67:55199
W20240502 01:58:28.998524 11602 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:28.998819 11602 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:28.998898 11602 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:29.001874 11602 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:29.001956 11602 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:29.002053 11602 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.68
I20240502 01:58:29.005671 11602 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.68:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.7.221.68
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--never_fsync=true
--heap_profile_path=/tmp/kudu.11602
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.68
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:29.006506 11602 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:29.007611 11602 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:29.015255 11607 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:29.015892 11608 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:29.016131 11610 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:29.016746 11602 server_base.cc:1034] running on GCE node
I20240502 01:58:29.017120 11602 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:29.017632 11602 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:29.018783 11602 hybrid_clock.cc:648] HybridClock initialized: now 1714615109018775 us; error 31 us; skew 500 ppm
I20240502 01:58:29.020656 11602 webserver.cc:460] Webserver started at http://127.7.221.68:37421/ using document root <none> and password file <none>
I20240502 01:58:29.021250 11602 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:29.021317 11602 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:29.021507 11602 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:29.023232 11602 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "089e23168d0b48a8baf89ee203e1b4fd"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:29.023703 11602 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "089e23168d0b48a8baf89ee203e1b4fd"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:29.026955 11602 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.004s sys 0.000s
I20240502 01:58:29.029156 11617 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:29.030099 11602 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:29.030205 11602 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "089e23168d0b48a8baf89ee203e1b4fd"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:29.030308 11602 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:29.048441 11602 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:29.048993 11602 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:29.049139 11602 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:29.049661 11602 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:29.050453 11602 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:29.050503 11602 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:29.050568 11602 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:29.050658 11602 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:29.058478 11602 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.68:43337
I20240502 01:58:29.058498 11733 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.68:43337 every 8 connection(s)
I20240502 01:58:29.059415 11602 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 01:58:29.060858 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 11602
I20240502 01:58:29.060971 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 01:58:29.063359 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.69:0
--local_ip_for_outbound_sockets=127.7.221.69
--webserver_interface=127.7.221.69
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--builtin_ntp_servers=127.7.221.84:39105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:29.069291 11734 heartbeater.cc:347] Connected to a master server at 127.7.221.126:32951
I20240502 01:58:29.069517 11734 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:29.070013 11734 heartbeater.cc:511] Master 127.7.221.126:32951 requested a full tablet report, sending...
I20240502 01:58:29.071009 11142 ts_manager.cc:194] Registered new tserver with Master: 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68:43337)
I20240502 01:58:29.071820 11142 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.68:41953
W20240502 01:58:29.168893 11737 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:29.169124 11737 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:29.169169 11737 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:29.172132 11737 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:29.172190 11737 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:29.172251 11737 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.69
I20240502 01:58:29.175700 11737 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.69:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.7.221.69
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--never_fsync=true
--heap_profile_path=/tmp/kudu.11737
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.69
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:29.176597 11737 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:29.177736 11737 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:29.185537 11742 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:29.186189 11743 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:29.186328 11745 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:29.187189 11737 server_base.cc:1034] running on GCE node
I20240502 01:58:29.187594 11737 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:29.188220 11737 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:29.189375 11737 hybrid_clock.cc:648] HybridClock initialized: now 1714615109189356 us; error 36 us; skew 500 ppm
I20240502 01:58:29.191262 11737 webserver.cc:460] Webserver started at http://127.7.221.69:46081/ using document root <none> and password file <none>
I20240502 01:58:29.191805 11737 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:29.191865 11737 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:29.192050 11737 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:29.193681 11737 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "0190cccef0ae48f3baa4123c0e4b49b0"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:29.194160 11737 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "0190cccef0ae48f3baa4123c0e4b49b0"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:29.197470 11737 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.003s
I20240502 01:58:29.199664 11752 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:29.200629 11737 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:29.200737 11737 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "0190cccef0ae48f3baa4123c0e4b49b0"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:29.200841 11737 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:29.224965 11737 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:29.225620 11737 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:29.225765 11737 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:29.226310 11737 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:29.227167 11737 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:29.227213 11737 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:29.227272 11737 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:29.227295 11737 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:29.236464 11737 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.69:35979
I20240502 01:58:29.236505 11869 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.69:35979 every 8 connection(s)
I20240502 01:58:29.237386 11737 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 01:58:29.240207 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 11737
I20240502 01:58:29.240305 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 01:58:29.242252 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.70:0
--local_ip_for_outbound_sockets=127.7.221.70
--webserver_interface=127.7.221.70
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--builtin_ntp_servers=127.7.221.84:39105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:29.246125 11870 heartbeater.cc:347] Connected to a master server at 127.7.221.126:32951
I20240502 01:58:29.246332 11870 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:29.246860 11870 heartbeater.cc:511] Master 127.7.221.126:32951 requested a full tablet report, sending...
I20240502 01:58:29.247888 11142 ts_manager.cc:194] Registered new tserver with Master: 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69:35979)
I20240502 01:58:29.248682 11142 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.69:48295
W20240502 01:58:29.348079 11873 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:29.348269 11873 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:29.348306 11873 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:29.350944 11873 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:29.350994 11873 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:29.351050 11873 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.70
I20240502 01:58:29.354580 11873 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.70:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.7.221.70
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--never_fsync=true
--heap_profile_path=/tmp/kudu.11873
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.70
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:29.356040 11873 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:29.357597 11873 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:29.367964 11881 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:29.368320 11878 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:29.368597 11879 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:29.369326 11873 server_base.cc:1034] running on GCE node
I20240502 01:58:29.369755 11873 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:29.370369 11873 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:29.371531 11873 hybrid_clock.cc:648] HybridClock initialized: now 1714615109371494 us; error 48 us; skew 500 ppm
I20240502 01:58:29.374056 11873 webserver.cc:460] Webserver started at http://127.7.221.70:36643/ using document root <none> and password file <none>
I20240502 01:58:29.374841 11873 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:29.374931 11873 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:29.375180 11873 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:29.377493 11873 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/instance:
uuid: "76bcefd6842e4c09ba16b8b6aa1ef662"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:29.378108 11873 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal/instance:
uuid: "76bcefd6842e4c09ba16b8b6aa1ef662"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:29.382292 11873 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.003s sys 0.000s
I20240502 01:58:29.384756 11887 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:29.386051 11873 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:29.386162 11873 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "76bcefd6842e4c09ba16b8b6aa1ef662"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:29.386279 11873 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:29.420365 11873 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:29.421073 11873 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:29.421244 11873 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:29.421825 11873 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:29.422776 11873 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:29.422824 11873 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:29.422890 11873 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:29.422922 11873 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:29.430985 11873 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.70:46849
I20240502 01:58:29.431035 12003 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.70:46849 every 8 connection(s)
I20240502 01:58:29.431869 11873 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 01:58:29.438853 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 11873
I20240502 01:58:29.438962 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240502 01:58:29.440312 12004 heartbeater.cc:347] Connected to a master server at 127.7.221.126:32951
I20240502 01:58:29.440560 12004 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:29.441205 12004 heartbeater.cc:511] Master 127.7.221.126:32951 requested a full tablet report, sending...
I20240502 01:58:29.441215 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.71:0
--local_ip_for_outbound_sockets=127.7.221.71
--webserver_interface=127.7.221.71
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--builtin_ntp_servers=127.7.221.84:39105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:29.442245 11143 ts_manager.cc:194] Registered new tserver with Master: 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70:46849)
I20240502 01:58:29.442821 11143 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.70:37779
W20240502 01:58:29.541713 12007 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:29.541937 12007 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:29.542012 12007 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:29.545123 12007 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:29.545192 12007 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:29.545292 12007 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.71
I20240502 01:58:29.548748 12007 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.71:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.7.221.71
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--never_fsync=true
--heap_profile_path=/tmp/kudu.12007
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.71
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:29.549562 12007 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:29.550576 12007 file_cache.cc:492] Constructed file cache file cache with capacity 419430
I20240502 01:58:29.556514 11333 heartbeater.cc:503] Master 127.7.221.126:32951 was elected leader, sending a full tablet report...
W20240502 01:58:29.558346 12012 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:29.558471 12015 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:29.558745 12013 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:29.559520 12007 server_base.cc:1034] running on GCE node
I20240502 01:58:29.559880 12007 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:29.560485 12007 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:29.561642 12007 hybrid_clock.cc:648] HybridClock initialized: now 1714615109561626 us; error 29 us; skew 500 ppm
I20240502 01:58:29.563645 12007 webserver.cc:460] Webserver started at http://127.7.221.71:37341/ using document root <none> and password file <none>
I20240502 01:58:29.564204 12007 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:29.564265 12007 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:29.564445 12007 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:29.566124 12007 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/instance:
uuid: "f271c45722d545f7b33fe7f407444dcb"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:29.566620 12007 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal/instance:
uuid: "f271c45722d545f7b33fe7f407444dcb"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:29.569850 12007 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:29.572037 12021 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:29.573040 12007 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:29.573146 12007 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "f271c45722d545f7b33fe7f407444dcb"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:29.573230 12007 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:29.584728 12007 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:29.585314 12007 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:29.585466 12007 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:29.585968 12007 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:29.586869 12007 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:29.586926 12007 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:29.586985 12007 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:29.587009 12007 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:29.595136 12007 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.71:38753
I20240502 01:58:29.595155 12136 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.71:38753 every 8 connection(s)
I20240502 01:58:29.596009 12007 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 01:58:29.596928 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 12007
I20240502 01:58:29.597050 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal/instance
I20240502 01:58:29.604166 12137 heartbeater.cc:347] Connected to a master server at 127.7.221.126:32951
I20240502 01:58:29.604391 12137 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:29.604888 12137 heartbeater.cc:511] Master 127.7.221.126:32951 requested a full tablet report, sending...
I20240502 01:58:29.605837 11143 ts_manager.cc:194] Registered new tserver with Master: f271c45722d545f7b33fe7f407444dcb (127.7.221.71:38753)
I20240502 01:58:29.606310 11143 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.71:59031
I20240502 01:58:29.610721 8053 external_mini_cluster.cc:930] 7 TS(s) registered with all masters
I20240502 01:58:29.618779 11143 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56106:
name: "TestTable"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 2
split_rows_range_bounds {
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
owner: "alice"
I20240502 01:58:29.635587 11936 tablet_service.cc:1467] Processing CreateTablet for tablet 1819f2f9c847491886d78211e3ca0fcc (DEFAULT_TABLE table=TestTable [id=e8e6680ba9ba4501a17920e8f48e41ea]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:29.635587 12070 tablet_service.cc:1467] Processing CreateTablet for tablet 1819f2f9c847491886d78211e3ca0fcc (DEFAULT_TABLE table=TestTable [id=e8e6680ba9ba4501a17920e8f48e41ea]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:29.636605 11936 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1819f2f9c847491886d78211e3ca0fcc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:29.636605 12070 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1819f2f9c847491886d78211e3ca0fcc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:29.642661 12154 tablet_bootstrap.cc:492] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662: Bootstrap starting.
I20240502 01:58:29.642661 12153 tablet_bootstrap.cc:492] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb: Bootstrap starting.
I20240502 01:58:29.644927 12154 tablet_bootstrap.cc:654] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:29.644927 12153 tablet_bootstrap.cc:654] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:29.645768 12153 log.cc:824] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:29.645768 12154 log.cc:824] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:29.647225 12154 tablet_bootstrap.cc:492] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662: No bootstrap required, opened a new log
I20240502 01:58:29.647243 12153 tablet_bootstrap.cc:492] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb: No bootstrap required, opened a new log
I20240502 01:58:29.647398 12154 ts_tablet_manager.cc:1367] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240502 01:58:29.647403 12153 ts_tablet_manager.cc:1367] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240502 01:58:29.650089 12153 raft_consensus.cc:348] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } }
I20240502 01:58:29.650072 12154 raft_consensus.cc:348] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } }
I20240502 01:58:29.650260 12153 raft_consensus.cc:374] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:29.650262 12154 raft_consensus.cc:374] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:29.650314 12154 raft_consensus.cc:729] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 76bcefd6842e4c09ba16b8b6aa1ef662, State: Initialized, Role: FOLLOWER
I20240502 01:58:29.650314 12153 raft_consensus.cc:729] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f271c45722d545f7b33fe7f407444dcb, State: Initialized, Role: FOLLOWER
I20240502 01:58:29.650705 12153 consensus_queue.cc:260] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } }
I20240502 01:58:29.650717 12154 consensus_queue.cc:260] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } }
I20240502 01:58:29.651298 12004 heartbeater.cc:503] Master 127.7.221.126:32951 was elected leader, sending a full tablet report...
I20240502 01:58:29.651329 12137 heartbeater.cc:503] Master 127.7.221.126:32951 was elected leader, sending a full tablet report...
I20240502 01:58:29.651556 12153 ts_tablet_manager.cc:1398] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb: Time spent starting tablet: real 0.004s user 0.004s sys 0.000s
I20240502 01:58:29.652412 12154 ts_tablet_manager.cc:1398] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s
W20240502 01:58:29.682940 12005 tablet.cc:2369] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 01:58:29.728086 11466 heartbeater.cc:503] Master 127.7.221.126:32951 was elected leader, sending a full tablet report...
I20240502 01:58:29.756210 12157 raft_consensus.cc:482] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:29.756389 12157 raft_consensus.cc:504] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } }
I20240502 01:58:29.757301 12157 leader_election.cc:288] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70:46849)
I20240502 01:58:29.760259 11956 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1819f2f9c847491886d78211e3ca0fcc" candidate_uuid: "f271c45722d545f7b33fe7f407444dcb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" is_pre_election: true
I20240502 01:58:29.760560 11956 raft_consensus.cc:2454] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f271c45722d545f7b33fe7f407444dcb in term 0.
I20240502 01:58:29.760957 12025 leader_election.cc:302] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76bcefd6842e4c09ba16b8b6aa1ef662, f271c45722d545f7b33fe7f407444dcb; no voters:
I20240502 01:58:29.761195 12157 raft_consensus.cc:2781] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:29.761266 12157 raft_consensus.cc:482] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:29.761301 12157 raft_consensus.cc:3037] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:29.762147 12157 raft_consensus.cc:504] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } }
I20240502 01:58:29.762393 12157 leader_election.cc:288] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [CANDIDATE]: Term 1 election: Requested vote from peers 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70:46849)
I20240502 01:58:29.762806 11956 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1819f2f9c847491886d78211e3ca0fcc" candidate_uuid: "f271c45722d545f7b33fe7f407444dcb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662"
I20240502 01:58:29.762923 11956 raft_consensus.cc:3037] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:29.763744 11956 raft_consensus.cc:2454] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f271c45722d545f7b33fe7f407444dcb in term 1.
I20240502 01:58:29.764070 12025 leader_election.cc:302] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76bcefd6842e4c09ba16b8b6aa1ef662, f271c45722d545f7b33fe7f407444dcb; no voters:
I20240502 01:58:29.764217 12157 raft_consensus.cc:2781] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:29.764448 12157 raft_consensus.cc:686] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 1 LEADER]: Becoming Leader. State: Replica: f271c45722d545f7b33fe7f407444dcb, State: Running, Role: LEADER
I20240502 01:58:29.764742 12157 consensus_queue.cc:237] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } }
I20240502 01:58:29.767611 11143 catalog_manager.cc:5484] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb reported cstate change: term changed from 0 to 1, leader changed from <none> to f271c45722d545f7b33fe7f407444dcb (127.7.221.71). New cstate: current_term: 1 leader_uuid: "f271c45722d545f7b33fe7f407444dcb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:29.779608 8053 external_mini_cluster.cc:930] 7 TS(s) registered with all masters
I20240502 01:58:29.780808 8053 ts_itest-base.cc:233] Waiting for 0 tablets on tserver a47375810d964f128aae1e2df1370b79 to finish bootstrapping
I20240502 01:58:29.786303 8053 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 6bf256be71684cdf87ef861e5890b614 to finish bootstrapping
I20240502 01:58:29.791363 8053 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 8c191817e0ce48b3b552af940f9d11ae to finish bootstrapping
I20240502 01:58:29.796066 8053 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 089e23168d0b48a8baf89ee203e1b4fd to finish bootstrapping
I20240502 01:58:29.800808 8053 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 0190cccef0ae48f3baa4123c0e4b49b0 to finish bootstrapping
I20240502 01:58:29.805836 8053 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 76bcefd6842e4c09ba16b8b6aa1ef662 to finish bootstrapping
I20240502 01:58:29.810029 8053 ts_itest-base.cc:233] Waiting for 1 tablets on tserver f271c45722d545f7b33fe7f407444dcb to finish bootstrapping
I20240502 01:58:29.814356 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 11602
I20240502 01:58:29.820628 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 11737
I20240502 01:58:29.826135 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 11873
I20240502 01:58:29.832008 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 12007
I20240502 01:58:29.898317 11599 heartbeater.cc:503] Master 127.7.221.126:32951 was elected leader, sending a full tablet report...
I20240502 01:58:33.588632 11143 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56106:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 2
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:33.601704 11531 tablet_service.cc:1467] Processing CreateTablet for tablet 706c5ef6b1b24ce28a70cef754b74d73 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d73cd318a885440582b140642ef6635e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:33.601663 11399 tablet_service.cc:1467] Processing CreateTablet for tablet 4ba576fba5b44e149c1c6a302248fd0f (DEFAULT_TABLE table=rebalance_test_table_0 [id=d73cd318a885440582b140642ef6635e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:33.601663 11267 tablet_service.cc:1467] Processing CreateTablet for tablet 4ba576fba5b44e149c1c6a302248fd0f (DEFAULT_TABLE table=rebalance_test_table_0 [id=d73cd318a885440582b140642ef6635e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:33.602543 11531 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 706c5ef6b1b24ce28a70cef754b74d73. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:33.602555 11267 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ba576fba5b44e149c1c6a302248fd0f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:33.602555 11399 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ba576fba5b44e149c1c6a302248fd0f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:33.604419 11530 tablet_service.cc:1467] Processing CreateTablet for tablet a1596ccad20249358ffbd0a4e3a5e1cb (DEFAULT_TABLE table=rebalance_test_table_0 [id=d73cd318a885440582b140642ef6635e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:33.604684 11530 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a1596ccad20249358ffbd0a4e3a5e1cb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:33.606163 11266 tablet_service.cc:1467] Processing CreateTablet for tablet 706c5ef6b1b24ce28a70cef754b74d73 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d73cd318a885440582b140642ef6635e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:33.606418 11266 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 706c5ef6b1b24ce28a70cef754b74d73. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:33.608767 12178 tablet_bootstrap.cc:492] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79: Bootstrap starting.
I20240502 01:58:33.608821 12179 tablet_bootstrap.cc:492] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae: Bootstrap starting.
I20240502 01:58:33.608929 11398 tablet_service.cc:1467] Processing CreateTablet for tablet a1596ccad20249358ffbd0a4e3a5e1cb (DEFAULT_TABLE table=rebalance_test_table_0 [id=d73cd318a885440582b140642ef6635e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:33.609170 11398 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a1596ccad20249358ffbd0a4e3a5e1cb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:33.610476 12179 tablet_bootstrap.cc:654] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:33.610659 12178 tablet_bootstrap.cc:654] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:33.611383 12177 tablet_bootstrap.cc:492] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614: Bootstrap starting.
I20240502 01:58:33.611419 12178 log.cc:824] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:33.611500 12179 log.cc:824] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:33.613225 12178 tablet_bootstrap.cc:492] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79: No bootstrap required, opened a new log
I20240502 01:58:33.613255 12179 tablet_bootstrap.cc:492] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae: No bootstrap required, opened a new log
I20240502 01:58:33.613263 12177 tablet_bootstrap.cc:654] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:33.613425 12179 ts_tablet_manager.cc:1367] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240502 01:58:33.613597 12178 ts_tablet_manager.cc:1367] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.004s
I20240502 01:58:33.614007 12177 log.cc:824] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:33.615635 12177 tablet_bootstrap.cc:492] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614: No bootstrap required, opened a new log
I20240502 01:58:33.615794 12177 ts_tablet_manager.cc:1367] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240502 01:58:33.616065 12179 raft_consensus.cc:348] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:33.616219 12179 raft_consensus.cc:374] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:33.616246 12179 raft_consensus.cc:729] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Initialized, Role: FOLLOWER
I20240502 01:58:33.616614 12179 consensus_queue.cc:260] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:33.617345 12179 ts_tablet_manager.cc:1398] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae: Time spent starting tablet: real 0.004s user 0.005s sys 0.000s
I20240502 01:58:33.617583 12179 tablet_bootstrap.cc:492] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae: Bootstrap starting.
I20240502 01:58:33.616977 12178 raft_consensus.cc:348] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:33.617878 12178 raft_consensus.cc:374] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:33.618137 12178 raft_consensus.cc:729] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a47375810d964f128aae1e2df1370b79, State: Initialized, Role: FOLLOWER
I20240502 01:58:33.618439 12179 tablet_bootstrap.cc:654] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:33.618626 12178 consensus_queue.cc:260] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:33.619413 12178 ts_tablet_manager.cc:1398] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79: Time spent starting tablet: real 0.006s user 0.003s sys 0.001s
I20240502 01:58:33.619689 12178 tablet_bootstrap.cc:492] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79: Bootstrap starting.
I20240502 01:58:33.620658 12178 tablet_bootstrap.cc:654] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:33.620791 12179 tablet_bootstrap.cc:492] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae: No bootstrap required, opened a new log
I20240502 01:58:33.620865 12179 ts_tablet_manager.cc:1367] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 01:58:33.620854 12177 raft_consensus.cc:348] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:33.620983 12177 raft_consensus.cc:374] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:33.621018 12177 raft_consensus.cc:729] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6bf256be71684cdf87ef861e5890b614, State: Initialized, Role: FOLLOWER
I20240502 01:58:33.621330 12177 consensus_queue.cc:260] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:33.621387 12179 raft_consensus.cc:348] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:33.621487 12179 raft_consensus.cc:374] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:33.621526 12179 raft_consensus.cc:729] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Initialized, Role: FOLLOWER
I20240502 01:58:33.621657 12179 consensus_queue.cc:260] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:33.621904 12179 ts_tablet_manager.cc:1398] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:33.621999 12178 tablet_bootstrap.cc:492] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79: No bootstrap required, opened a new log
I20240502 01:58:33.622068 12178 ts_tablet_manager.cc:1367] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.001s
I20240502 01:58:33.622104 12177 ts_tablet_manager.cc:1398] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614: Time spent starting tablet: real 0.006s user 0.004s sys 0.000s
I20240502 01:58:33.622509 12178 raft_consensus.cc:348] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:33.622620 12178 raft_consensus.cc:374] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:33.622655 12178 raft_consensus.cc:729] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a47375810d964f128aae1e2df1370b79, State: Initialized, Role: FOLLOWER
I20240502 01:58:33.622762 12178 consensus_queue.cc:260] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:33.623004 12178 ts_tablet_manager.cc:1398] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:33.623248 12177 tablet_bootstrap.cc:492] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614: Bootstrap starting.
I20240502 01:58:33.624239 12177 tablet_bootstrap.cc:654] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:33.625226 12177 tablet_bootstrap.cc:492] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614: No bootstrap required, opened a new log
I20240502 01:58:33.625286 12177 ts_tablet_manager.cc:1367] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:33.625644 12177 raft_consensus.cc:348] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:33.625715 12177 raft_consensus.cc:374] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:33.625767 12177 raft_consensus.cc:729] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6bf256be71684cdf87ef861e5890b614, State: Initialized, Role: FOLLOWER
I20240502 01:58:33.625866 12177 consensus_queue.cc:260] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:33.626116 12177 ts_tablet_manager.cc:1398] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
W20240502 01:58:33.640838 11600 tablet.cc:2369] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 01:58:33.695681 12185 raft_consensus.cc:482] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:33.695884 12185 raft_consensus.cc:504] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:33.696859 12185 leader_election.cc:288] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:33.700301 11552 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "706c5ef6b1b24ce28a70cef754b74d73" candidate_uuid: "a47375810d964f128aae1e2df1370b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c191817e0ce48b3b552af940f9d11ae" is_pre_election: true
I20240502 01:58:33.700556 11552 raft_consensus.cc:2454] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a47375810d964f128aae1e2df1370b79 in term 0.
I20240502 01:58:33.700932 11221 leader_election.cc:302] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 8c191817e0ce48b3b552af940f9d11ae, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:33.701166 12185 raft_consensus.cc:2781] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:33.701238 12185 raft_consensus.cc:482] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:33.701265 12185 raft_consensus.cc:3037] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:33.702189 12185 raft_consensus.cc:504] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:33.702457 12185 leader_election.cc:288] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 election: Requested vote from peers 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:33.702867 11552 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "706c5ef6b1b24ce28a70cef754b74d73" candidate_uuid: "a47375810d964f128aae1e2df1370b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c191817e0ce48b3b552af940f9d11ae"
I20240502 01:58:33.703001 11552 raft_consensus.cc:3037] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:33.703903 11552 raft_consensus.cc:2454] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a47375810d964f128aae1e2df1370b79 in term 1.
I20240502 01:58:33.704182 11221 leader_election.cc:302] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 8c191817e0ce48b3b552af940f9d11ae, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:33.704365 12185 raft_consensus.cc:2781] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:33.704609 12185 raft_consensus.cc:686] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Becoming Leader. State: Replica: a47375810d964f128aae1e2df1370b79, State: Running, Role: LEADER
I20240502 01:58:33.704898 12185 consensus_queue.cc:237] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:33.707659 11143 catalog_manager.cc:5484] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 reported cstate change: term changed from 0 to 1, leader changed from <none> to a47375810d964f128aae1e2df1370b79 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: HEALTHY } } }
W20240502 01:58:33.721012 11467 tablet.cc:2369] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 01:58:33.789923 12184 raft_consensus.cc:482] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:33.790066 12184 raft_consensus.cc:504] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:33.791075 12184 leader_election.cc:288] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:33.794797 11420 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a1596ccad20249358ffbd0a4e3a5e1cb" candidate_uuid: "8c191817e0ce48b3b552af940f9d11ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf256be71684cdf87ef861e5890b614" is_pre_election: true
I20240502 01:58:33.795083 11420 raft_consensus.cc:2454] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8c191817e0ce48b3b552af940f9d11ae in term 0.
I20240502 01:58:33.795435 11486 leader_election.cc:302] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, 8c191817e0ce48b3b552af940f9d11ae; no voters:
I20240502 01:58:33.795657 12184 raft_consensus.cc:2781] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:33.795723 12184 raft_consensus.cc:482] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:33.795776 12184 raft_consensus.cc:3037] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Advancing to term 1
W20240502 01:58:33.796330 11334 tablet.cc:2369] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 01:58:33.796734 12184 raft_consensus.cc:504] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:33.797027 12184 leader_election.cc:288] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 election: Requested vote from peers 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:33.797380 11420 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a1596ccad20249358ffbd0a4e3a5e1cb" candidate_uuid: "8c191817e0ce48b3b552af940f9d11ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf256be71684cdf87ef861e5890b614"
I20240502 01:58:33.797483 11420 raft_consensus.cc:3037] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:33.798270 11420 raft_consensus.cc:2454] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8c191817e0ce48b3b552af940f9d11ae in term 1.
I20240502 01:58:33.798527 11486 leader_election.cc:302] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, 8c191817e0ce48b3b552af940f9d11ae; no voters:
I20240502 01:58:33.798733 12184 raft_consensus.cc:2781] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:33.798930 12184 raft_consensus.cc:686] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Becoming Leader. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Running, Role: LEADER
I20240502 01:58:33.799204 12184 consensus_queue.cc:237] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:33.801867 11143 catalog_manager.cc:5484] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: term changed from 0 to 1, leader changed from <none> to 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67). New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } }
I20240502 01:58:33.944254 12188 raft_consensus.cc:482] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:33.944442 12188 raft_consensus.cc:504] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:33.945451 12188 leader_election.cc:288] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857)
I20240502 01:58:33.948840 11287 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ba576fba5b44e149c1c6a302248fd0f" candidate_uuid: "6bf256be71684cdf87ef861e5890b614" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a47375810d964f128aae1e2df1370b79" is_pre_election: true
I20240502 01:58:33.949046 11287 raft_consensus.cc:2454] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6bf256be71684cdf87ef861e5890b614 in term 0.
I20240502 01:58:33.949432 11353 leader_election.cc:302] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:33.949676 12188 raft_consensus.cc:2781] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:33.949736 12188 raft_consensus.cc:482] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:33.949790 12188 raft_consensus.cc:3037] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:33.950649 12188 raft_consensus.cc:504] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:33.950917 12188 leader_election.cc:288] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [CANDIDATE]: Term 1 election: Requested vote from peers a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857)
I20240502 01:58:33.951366 11287 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ba576fba5b44e149c1c6a302248fd0f" candidate_uuid: "6bf256be71684cdf87ef861e5890b614" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a47375810d964f128aae1e2df1370b79"
I20240502 01:58:33.951470 11287 raft_consensus.cc:3037] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:33.952210 11287 raft_consensus.cc:2454] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6bf256be71684cdf87ef861e5890b614 in term 1.
I20240502 01:58:33.952484 11353 leader_election.cc:302] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:33.952636 12188 raft_consensus.cc:2781] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:33.952840 12188 raft_consensus.cc:686] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [term 1 LEADER]: Becoming Leader. State: Replica: 6bf256be71684cdf87ef861e5890b614, State: Running, Role: LEADER
I20240502 01:58:33.953176 12188 consensus_queue.cc:237] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:33.956466 11143 catalog_manager.cc:5484] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6bf256be71684cdf87ef861e5890b614 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "6bf256be71684cdf87ef861e5890b614" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:34.154467 12199 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:34.154732 12199 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:34.157593 12199 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_0'
I20240502 01:58:34.192206 12185 consensus_queue.cc:1035] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:34.260599 11420 raft_consensus.cc:1261] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:34.261196 12184 consensus_queue.cc:1035] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:34.268088 11287 raft_consensus.cc:1261] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:34.268787 12188 consensus_queue.cc:1035] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:34.276587 12234 mvcc.cc:205] Tried to move back new op lower bound from 7023063507999842304 to 7023063506123468800. Current Snapshot: MvccSnapshot[applied={T|T < 7023063507999842304}]
I20240502 01:58:34.277644 12238 mvcc.cc:205] Tried to move back new op lower bound from 7023063507999842304 to 7023063506123468800. Current Snapshot: MvccSnapshot[applied={T|T < 7023063507999842304}]
I20240502 01:58:34.288741 12222 mvcc.cc:205] Tried to move back new op lower bound from 7023063508037304320 to 7023063506754535424. Current Snapshot: MvccSnapshot[applied={T|T < 7023063508037304320}]
INSERT report
rows total: 2000
time total: 99.6237 ms
time per row: 0.0498119 ms
I20240502 01:58:34.308192 11143 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56106:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 2
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:34.313491 11267 tablet_service.cc:1467] Processing CreateTablet for tablet a6b4f69afc6149feac9ca1e82ae995ff (DEFAULT_TABLE table=rebalance_test_table_1 [id=cb616922efbb491588387c177fbfe8df]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:34.313653 11398 tablet_service.cc:1467] Processing CreateTablet for tablet a6b4f69afc6149feac9ca1e82ae995ff (DEFAULT_TABLE table=rebalance_test_table_1 [id=cb616922efbb491588387c177fbfe8df]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:34.313820 11267 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6b4f69afc6149feac9ca1e82ae995ff. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:34.313979 11398 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6b4f69afc6149feac9ca1e82ae995ff. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:34.314337 11530 tablet_service.cc:1467] Processing CreateTablet for tablet 9375f41e0aad43d3a9d6f5ef7ac73880 (DEFAULT_TABLE table=rebalance_test_table_1 [id=cb616922efbb491588387c177fbfe8df]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:34.314373 11531 tablet_service.cc:1467] Processing CreateTablet for tablet 91e43d66f1664c7db8f1f16f623640be (DEFAULT_TABLE table=rebalance_test_table_1 [id=cb616922efbb491588387c177fbfe8df]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:34.314831 11530 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9375f41e0aad43d3a9d6f5ef7ac73880. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:34.314882 11399 tablet_service.cc:1467] Processing CreateTablet for tablet 91e43d66f1664c7db8f1f16f623640be (DEFAULT_TABLE table=rebalance_test_table_1 [id=cb616922efbb491588387c177fbfe8df]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:34.315093 11399 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 91e43d66f1664c7db8f1f16f623640be. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:34.316449 11267 tablet_service.cc:1467] Processing CreateTablet for tablet 9375f41e0aad43d3a9d6f5ef7ac73880 (DEFAULT_TABLE table=rebalance_test_table_1 [id=cb616922efbb491588387c177fbfe8df]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:34.316653 11267 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9375f41e0aad43d3a9d6f5ef7ac73880. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:34.316948 12250 tablet_bootstrap.cc:492] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614: Bootstrap starting.
I20240502 01:58:34.317171 11531 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 91e43d66f1664c7db8f1f16f623640be. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:34.318157 12249 tablet_bootstrap.cc:492] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79: Bootstrap starting.
I20240502 01:58:34.318195 12250 tablet_bootstrap.cc:654] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:34.318953 12249 tablet_bootstrap.cc:654] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:34.319352 12251 tablet_bootstrap.cc:492] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae: Bootstrap starting.
I20240502 01:58:34.319900 12249 tablet_bootstrap.cc:492] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79: No bootstrap required, opened a new log
I20240502 01:58:34.319983 12249 ts_tablet_manager.cc:1367] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:34.320071 12251 tablet_bootstrap.cc:654] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:34.320375 12249 raft_consensus.cc:348] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:34.320458 12249 raft_consensus.cc:374] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:34.320478 12249 raft_consensus.cc:729] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a47375810d964f128aae1e2df1370b79, State: Initialized, Role: FOLLOWER
I20240502 01:58:34.320593 12249 consensus_queue.cc:260] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:34.320928 12249 ts_tablet_manager.cc:1398] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:34.321081 12250 tablet_bootstrap.cc:492] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614: No bootstrap required, opened a new log
I20240502 01:58:34.321089 12249 tablet_bootstrap.cc:492] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79: Bootstrap starting.
I20240502 01:58:34.321097 12251 tablet_bootstrap.cc:492] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae: No bootstrap required, opened a new log
I20240502 01:58:34.321153 12251 ts_tablet_manager.cc:1367] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:34.321153 12250 ts_tablet_manager.cc:1367] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:58:34.321523 12250 raft_consensus.cc:348] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:34.321523 12251 raft_consensus.cc:348] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:34.321605 12251 raft_consensus.cc:374] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:34.321605 12250 raft_consensus.cc:374] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:34.321633 12251 raft_consensus.cc:729] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Initialized, Role: FOLLOWER
I20240502 01:58:34.321633 12250 raft_consensus.cc:729] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6bf256be71684cdf87ef861e5890b614, State: Initialized, Role: FOLLOWER
I20240502 01:58:34.321803 12250 consensus_queue.cc:260] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:34.321791 12251 consensus_queue.cc:260] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:34.321854 12249 tablet_bootstrap.cc:654] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:34.322098 12251 ts_tablet_manager.cc:1398] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:34.322232 12251 tablet_bootstrap.cc:492] T 91e43d66f1664c7db8f1f16f623640be P 8c191817e0ce48b3b552af940f9d11ae: Bootstrap starting.
I20240502 01:58:34.323120 12251 tablet_bootstrap.cc:654] T 91e43d66f1664c7db8f1f16f623640be P 8c191817e0ce48b3b552af940f9d11ae: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:34.323268 12250 ts_tablet_manager.cc:1398] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:34.323441 12250 tablet_bootstrap.cc:492] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614: Bootstrap starting.
I20240502 01:58:34.324357 12250 tablet_bootstrap.cc:654] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:34.324405 12251 tablet_bootstrap.cc:492] T 91e43d66f1664c7db8f1f16f623640be P 8c191817e0ce48b3b552af940f9d11ae: No bootstrap required, opened a new log
I20240502 01:58:34.324483 12251 ts_tablet_manager.cc:1367] T 91e43d66f1664c7db8f1f16f623640be P 8c191817e0ce48b3b552af940f9d11ae: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 01:58:34.324921 12251 raft_consensus.cc:348] T 91e43d66f1664c7db8f1f16f623640be P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:34.325028 12251 raft_consensus.cc:374] T 91e43d66f1664c7db8f1f16f623640be P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:34.325069 12251 raft_consensus.cc:729] T 91e43d66f1664c7db8f1f16f623640be P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Initialized, Role: FOLLOWER
I20240502 01:58:34.325219 12251 consensus_queue.cc:260] T 91e43d66f1664c7db8f1f16f623640be P 8c191817e0ce48b3b552af940f9d11ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:34.325533 12249 tablet_bootstrap.cc:492] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79: No bootstrap required, opened a new log
I20240502 01:58:34.325543 12250 tablet_bootstrap.cc:492] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614: No bootstrap required, opened a new log
I20240502 01:58:34.325567 12251 ts_tablet_manager.cc:1398] T 91e43d66f1664c7db8f1f16f623640be P 8c191817e0ce48b3b552af940f9d11ae: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:34.325603 12249 ts_tablet_manager.cc:1367] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 01:58:34.325616 12250 ts_tablet_manager.cc:1367] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:34.325925 12249 raft_consensus.cc:348] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:34.325991 12249 raft_consensus.cc:374] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:34.326009 12249 raft_consensus.cc:729] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a47375810d964f128aae1e2df1370b79, State: Initialized, Role: FOLLOWER
I20240502 01:58:34.326042 12250 raft_consensus.cc:348] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:34.326112 12249 consensus_queue.cc:260] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:34.326136 12250 raft_consensus.cc:374] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:34.326164 12250 raft_consensus.cc:729] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6bf256be71684cdf87ef861e5890b614, State: Initialized, Role: FOLLOWER
I20240502 01:58:34.326282 12250 consensus_queue.cc:260] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:34.326602 12249 ts_tablet_manager.cc:1398] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:34.326558 12250 ts_tablet_manager.cc:1398] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:34.405263 12197 raft_consensus.cc:482] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:34.405458 12197 raft_consensus.cc:504] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:34.405944 12197 leader_election.cc:288] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:34.407485 12191 raft_consensus.cc:482] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:34.407595 12191 raft_consensus.cc:504] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:34.407900 12191 leader_election.cc:288] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:34.408603 11552 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9375f41e0aad43d3a9d6f5ef7ac73880" candidate_uuid: "a47375810d964f128aae1e2df1370b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c191817e0ce48b3b552af940f9d11ae" is_pre_election: true
I20240502 01:58:34.408739 11552 raft_consensus.cc:2454] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a47375810d964f128aae1e2df1370b79 in term 0.
I20240502 01:58:34.409070 11221 leader_election.cc:302] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 8c191817e0ce48b3b552af940f9d11ae, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:34.409281 12191 raft_consensus.cc:2781] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:34.409356 12191 raft_consensus.cc:482] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:34.409401 12191 raft_consensus.cc:3037] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:34.409716 11552 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "91e43d66f1664c7db8f1f16f623640be" candidate_uuid: "6bf256be71684cdf87ef861e5890b614" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c191817e0ce48b3b552af940f9d11ae" is_pre_election: true
I20240502 01:58:34.409834 11552 raft_consensus.cc:2454] T 91e43d66f1664c7db8f1f16f623640be P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6bf256be71684cdf87ef861e5890b614 in term 0.
I20240502 01:58:34.410095 11354 leader_election.cc:302] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, 8c191817e0ce48b3b552af940f9d11ae; no voters:
I20240502 01:58:34.410297 12197 raft_consensus.cc:2781] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:34.410372 12197 raft_consensus.cc:482] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:34.410403 12197 raft_consensus.cc:3037] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:34.412039 12191 raft_consensus.cc:504] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:34.412324 12191 leader_election.cc:288] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 election: Requested vote from peers 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:34.412511 12197 raft_consensus.cc:504] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:34.412626 11552 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9375f41e0aad43d3a9d6f5ef7ac73880" candidate_uuid: "a47375810d964f128aae1e2df1370b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c191817e0ce48b3b552af940f9d11ae"
I20240502 01:58:34.412734 11552 raft_consensus.cc:3037] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:34.412807 12197 leader_election.cc:288] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [CANDIDATE]: Term 1 election: Requested vote from peers 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:34.413195 11551 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "91e43d66f1664c7db8f1f16f623640be" candidate_uuid: "6bf256be71684cdf87ef861e5890b614" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c191817e0ce48b3b552af940f9d11ae"
I20240502 01:58:34.413305 11551 raft_consensus.cc:3037] T 91e43d66f1664c7db8f1f16f623640be P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:34.413789 11552 raft_consensus.cc:2454] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a47375810d964f128aae1e2df1370b79 in term 1.
I20240502 01:58:34.414079 11221 leader_election.cc:302] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 8c191817e0ce48b3b552af940f9d11ae, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:34.414302 12191 raft_consensus.cc:2781] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:34.414301 11551 raft_consensus.cc:2454] T 91e43d66f1664c7db8f1f16f623640be P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6bf256be71684cdf87ef861e5890b614 in term 1.
I20240502 01:58:34.414398 12191 raft_consensus.cc:686] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Becoming Leader. State: Replica: a47375810d964f128aae1e2df1370b79, State: Running, Role: LEADER
I20240502 01:58:34.414554 11354 leader_election.cc:302] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, 8c191817e0ce48b3b552af940f9d11ae; no voters:
I20240502 01:58:34.414534 12191 consensus_queue.cc:237] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:34.414772 12197 raft_consensus.cc:2781] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:34.414851 12197 raft_consensus.cc:686] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [term 1 LEADER]: Becoming Leader. State: Replica: 6bf256be71684cdf87ef861e5890b614, State: Running, Role: LEADER
I20240502 01:58:34.414956 12197 consensus_queue.cc:237] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:34.416144 11142 catalog_manager.cc:5484] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 reported cstate change: term changed from 0 to 1, leader changed from <none> to a47375810d964f128aae1e2df1370b79 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:34.416383 11141 catalog_manager.cc:5484] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6bf256be71684cdf87ef861e5890b614 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "6bf256be71684cdf87ef861e5890b614" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:34.615041 12191 raft_consensus.cc:482] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:34.615183 12191 raft_consensus.cc:504] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:34.615594 12191 leader_election.cc:288] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:34.620291 11420 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6b4f69afc6149feac9ca1e82ae995ff" candidate_uuid: "a47375810d964f128aae1e2df1370b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf256be71684cdf87ef861e5890b614" is_pre_election: true
I20240502 01:58:34.620460 11420 raft_consensus.cc:2454] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a47375810d964f128aae1e2df1370b79 in term 0.
I20240502 01:58:34.620774 11221 leader_election.cc:302] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:34.621006 12191 raft_consensus.cc:2781] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:34.621048 12191 raft_consensus.cc:482] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:34.621088 12191 raft_consensus.cc:3037] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:34.621960 12191 raft_consensus.cc:504] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:34.622237 12191 leader_election.cc:288] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 election: Requested vote from peers 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:34.622633 11420 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6b4f69afc6149feac9ca1e82ae995ff" candidate_uuid: "a47375810d964f128aae1e2df1370b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf256be71684cdf87ef861e5890b614"
I20240502 01:58:34.622783 11420 raft_consensus.cc:3037] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:34.623499 11420 raft_consensus.cc:2454] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a47375810d964f128aae1e2df1370b79 in term 1.
I20240502 01:58:34.623801 11221 leader_election.cc:302] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:34.624007 12191 raft_consensus.cc:2781] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:34.624111 12191 raft_consensus.cc:686] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Becoming Leader. State: Replica: a47375810d964f128aae1e2df1370b79, State: Running, Role: LEADER
I20240502 01:58:34.624243 12191 consensus_queue.cc:237] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:34.626024 11141 catalog_manager.cc:5484] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 reported cstate change: term changed from 0 to 1, leader changed from <none> to a47375810d964f128aae1e2df1370b79 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:34.789095 12263 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:34.789342 12263 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:34.792310 12263 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_1'
I20240502 01:58:34.867940 11551 raft_consensus.cc:1261] T 91e43d66f1664c7db8f1f16f623640be P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:34.868419 12188 consensus_queue.cc:1035] T 91e43d66f1664c7db8f1f16f623640be P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:34.869838 11420 raft_consensus.cc:1261] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:34.870245 12191 consensus_queue.cc:1035] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:34.871842 11552 raft_consensus.cc:1261] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:34.873212 12191 consensus_queue.cc:1035] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 90.6108 ms
time per row: 0.0453054 ms
I20240502 01:58:34.922963 11139 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56106:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 2
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:34.928090 11399 tablet_service.cc:1467] Processing CreateTablet for tablet 5ddde4eecc19461db50d15139066ea31 (DEFAULT_TABLE table=rebalance_test_table_2 [id=3953bb3e36b342bf9945db89810e6cf0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:34.928239 11267 tablet_service.cc:1467] Processing CreateTablet for tablet 5ddde4eecc19461db50d15139066ea31 (DEFAULT_TABLE table=rebalance_test_table_2 [id=3953bb3e36b342bf9945db89810e6cf0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:34.928296 11531 tablet_service.cc:1467] Processing CreateTablet for tablet 4cef15d5c5fc41cbb55d184731eff9e2 (DEFAULT_TABLE table=rebalance_test_table_2 [id=3953bb3e36b342bf9945db89810e6cf0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:34.928438 11399 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5ddde4eecc19461db50d15139066ea31. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:34.928553 11531 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4cef15d5c5fc41cbb55d184731eff9e2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:34.928555 11267 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5ddde4eecc19461db50d15139066ea31. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:34.930932 12312 tablet_bootstrap.cc:492] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae: Bootstrap starting.
I20240502 01:58:34.931078 12313 tablet_bootstrap.cc:492] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79: Bootstrap starting.
I20240502 01:58:34.931280 11266 tablet_service.cc:1467] Processing CreateTablet for tablet 4cef15d5c5fc41cbb55d184731eff9e2 (DEFAULT_TABLE table=rebalance_test_table_2 [id=3953bb3e36b342bf9945db89810e6cf0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:34.931514 11266 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4cef15d5c5fc41cbb55d184731eff9e2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:34.931905 12312 tablet_bootstrap.cc:654] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:34.931972 12313 tablet_bootstrap.cc:654] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:34.932787 11399 tablet_service.cc:1467] Processing CreateTablet for tablet 2dd97f65baa14cd9b3010b5e20592f1f (DEFAULT_TABLE table=rebalance_test_table_2 [id=3953bb3e36b342bf9945db89810e6cf0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:34.932910 12312 tablet_bootstrap.cc:492] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae: No bootstrap required, opened a new log
I20240502 01:58:34.932998 12312 ts_tablet_manager.cc:1367] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:34.933033 11399 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2dd97f65baa14cd9b3010b5e20592f1f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:34.933599 12312 raft_consensus.cc:348] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:34.933701 12312 raft_consensus.cc:374] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:34.933750 12312 raft_consensus.cc:729] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Initialized, Role: FOLLOWER
I20240502 01:58:34.933873 12312 consensus_queue.cc:260] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:34.934109 12312 ts_tablet_manager.cc:1398] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:34.935067 12313 tablet_bootstrap.cc:492] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79: No bootstrap required, opened a new log
I20240502 01:58:34.935163 12313 ts_tablet_manager.cc:1367] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:58:34.935349 11531 tablet_service.cc:1467] Processing CreateTablet for tablet 2dd97f65baa14cd9b3010b5e20592f1f (DEFAULT_TABLE table=rebalance_test_table_2 [id=3953bb3e36b342bf9945db89810e6cf0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:34.935578 11531 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2dd97f65baa14cd9b3010b5e20592f1f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:34.935621 12313 raft_consensus.cc:348] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:34.935716 12313 raft_consensus.cc:374] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:34.935751 12311 tablet_bootstrap.cc:492] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614: Bootstrap starting.
I20240502 01:58:34.935746 12313 raft_consensus.cc:729] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a47375810d964f128aae1e2df1370b79, State: Initialized, Role: FOLLOWER
I20240502 01:58:34.935883 12313 consensus_queue.cc:260] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:34.936141 12313 ts_tablet_manager.cc:1398] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:34.936295 12313 tablet_bootstrap.cc:492] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79: Bootstrap starting.
I20240502 01:58:34.937062 12311 tablet_bootstrap.cc:654] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:34.937084 12313 tablet_bootstrap.cc:654] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:34.937528 12312 tablet_bootstrap.cc:492] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae: Bootstrap starting.
I20240502 01:58:34.938405 12312 tablet_bootstrap.cc:654] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:34.938545 12313 tablet_bootstrap.cc:492] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79: No bootstrap required, opened a new log
I20240502 01:58:34.938545 12311 tablet_bootstrap.cc:492] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614: No bootstrap required, opened a new log
I20240502 01:58:34.938648 12311 ts_tablet_manager.cc:1367] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:34.938648 12313 ts_tablet_manager.cc:1367] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:34.939064 12313 raft_consensus.cc:348] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:34.939074 12311 raft_consensus.cc:348] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:34.939168 12311 raft_consensus.cc:374] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:34.939168 12313 raft_consensus.cc:374] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:34.939201 12313 raft_consensus.cc:729] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a47375810d964f128aae1e2df1370b79, State: Initialized, Role: FOLLOWER
I20240502 01:58:34.939201 12311 raft_consensus.cc:729] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6bf256be71684cdf87ef861e5890b614, State: Initialized, Role: FOLLOWER
I20240502 01:58:34.939324 12311 consensus_queue.cc:260] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:34.939345 12313 consensus_queue.cc:260] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:34.939574 12311 ts_tablet_manager.cc:1398] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:34.939574 12313 ts_tablet_manager.cc:1398] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:34.939738 12311 tablet_bootstrap.cc:492] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614: Bootstrap starting.
I20240502 01:58:34.939953 12312 tablet_bootstrap.cc:492] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae: No bootstrap required, opened a new log
I20240502 01:58:34.940058 12312 ts_tablet_manager.cc:1367] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:34.940498 12311 tablet_bootstrap.cc:654] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:34.940459 12312 raft_consensus.cc:348] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:34.940558 12312 raft_consensus.cc:374] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:34.940594 12312 raft_consensus.cc:729] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Initialized, Role: FOLLOWER
I20240502 01:58:34.940721 12312 consensus_queue.cc:260] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:34.940977 12312 ts_tablet_manager.cc:1398] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:34.942651 12311 tablet_bootstrap.cc:492] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614: No bootstrap required, opened a new log
I20240502 01:58:34.942739 12311 ts_tablet_manager.cc:1367] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:34.943217 12311 raft_consensus.cc:348] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:34.943305 12311 raft_consensus.cc:374] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:34.943339 12311 raft_consensus.cc:729] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6bf256be71684cdf87ef861e5890b614, State: Initialized, Role: FOLLOWER
I20240502 01:58:34.943465 12311 consensus_queue.cc:260] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:34.943677 12311 ts_tablet_manager.cc:1398] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:35.020679 12287 raft_consensus.cc:482] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:35.020815 12287 raft_consensus.cc:504] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:35.021274 12287 leader_election.cc:288] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857)
I20240502 01:58:35.024870 11287 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4cef15d5c5fc41cbb55d184731eff9e2" candidate_uuid: "8c191817e0ce48b3b552af940f9d11ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a47375810d964f128aae1e2df1370b79" is_pre_election: true
I20240502 01:58:35.025031 11287 raft_consensus.cc:2454] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8c191817e0ce48b3b552af940f9d11ae in term 0.
I20240502 01:58:35.025399 11485 leader_election.cc:302] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 8c191817e0ce48b3b552af940f9d11ae, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:35.025626 12287 raft_consensus.cc:2781] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:35.025696 12287 raft_consensus.cc:482] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:35.025727 12287 raft_consensus.cc:3037] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:35.026578 12287 raft_consensus.cc:504] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:35.026894 12287 leader_election.cc:288] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 election: Requested vote from peers a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857)
I20240502 01:58:35.027292 11287 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4cef15d5c5fc41cbb55d184731eff9e2" candidate_uuid: "8c191817e0ce48b3b552af940f9d11ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a47375810d964f128aae1e2df1370b79"
I20240502 01:58:35.027398 11287 raft_consensus.cc:3037] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:35.028239 11287 raft_consensus.cc:2454] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8c191817e0ce48b3b552af940f9d11ae in term 1.
I20240502 01:58:35.028509 11485 leader_election.cc:302] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 8c191817e0ce48b3b552af940f9d11ae, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:35.028669 12287 raft_consensus.cc:2781] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:35.028887 12287 raft_consensus.cc:686] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Becoming Leader. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Running, Role: LEADER
I20240502 01:58:35.029057 12287 consensus_queue.cc:237] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:35.030505 11139 catalog_manager.cc:5484] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: term changed from 0 to 1, leader changed from <none> to 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67). New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:35.057253 12287 raft_consensus.cc:482] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:35.057396 12287 raft_consensus.cc:504] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:35.057708 12287 leader_election.cc:288] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:35.058193 11420 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2dd97f65baa14cd9b3010b5e20592f1f" candidate_uuid: "8c191817e0ce48b3b552af940f9d11ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf256be71684cdf87ef861e5890b614" is_pre_election: true
I20240502 01:58:35.058329 11420 raft_consensus.cc:2454] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8c191817e0ce48b3b552af940f9d11ae in term 0.
I20240502 01:58:35.058656 11486 leader_election.cc:302] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, 8c191817e0ce48b3b552af940f9d11ae; no voters:
I20240502 01:58:35.058818 12287 raft_consensus.cc:2781] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:35.058877 12287 raft_consensus.cc:482] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:35.058912 12287 raft_consensus.cc:3037] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:35.059682 12287 raft_consensus.cc:504] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:35.059981 12287 leader_election.cc:288] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 election: Requested vote from peers 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:35.060351 11420 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2dd97f65baa14cd9b3010b5e20592f1f" candidate_uuid: "8c191817e0ce48b3b552af940f9d11ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf256be71684cdf87ef861e5890b614"
I20240502 01:58:35.060456 11420 raft_consensus.cc:3037] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:35.061158 11420 raft_consensus.cc:2454] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8c191817e0ce48b3b552af940f9d11ae in term 1.
I20240502 01:58:35.061426 11486 leader_election.cc:302] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, 8c191817e0ce48b3b552af940f9d11ae; no voters:
I20240502 01:58:35.061589 12287 raft_consensus.cc:2781] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:35.061668 12287 raft_consensus.cc:686] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Becoming Leader. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Running, Role: LEADER
I20240502 01:58:35.062000 12287 consensus_queue.cc:237] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:35.063592 11139 catalog_manager.cc:5484] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: term changed from 0 to 1, leader changed from <none> to 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67). New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:35.196925 12310 raft_consensus.cc:482] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:35.197086 12310 raft_consensus.cc:504] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:35.197510 12310 leader_election.cc:288] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:35.198024 11420 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5ddde4eecc19461db50d15139066ea31" candidate_uuid: "a47375810d964f128aae1e2df1370b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf256be71684cdf87ef861e5890b614" is_pre_election: true
I20240502 01:58:35.198163 11420 raft_consensus.cc:2454] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a47375810d964f128aae1e2df1370b79 in term 0.
I20240502 01:58:35.198474 11221 leader_election.cc:302] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:35.198654 12310 raft_consensus.cc:2781] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:35.198707 12310 raft_consensus.cc:482] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:35.198732 12310 raft_consensus.cc:3037] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:35.199597 12310 raft_consensus.cc:504] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:35.199877 12310 leader_election.cc:288] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 election: Requested vote from peers 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:35.200281 11420 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5ddde4eecc19461db50d15139066ea31" candidate_uuid: "a47375810d964f128aae1e2df1370b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf256be71684cdf87ef861e5890b614"
I20240502 01:58:35.200394 11420 raft_consensus.cc:3037] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:35.201148 11420 raft_consensus.cc:2454] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a47375810d964f128aae1e2df1370b79 in term 1.
I20240502 01:58:35.201443 11221 leader_election.cc:302] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:35.201609 12310 raft_consensus.cc:2781] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:35.201683 12310 raft_consensus.cc:686] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Becoming Leader. State: Replica: a47375810d964f128aae1e2df1370b79, State: Running, Role: LEADER
I20240502 01:58:35.201821 12310 consensus_queue.cc:237] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:35.203389 11139 catalog_manager.cc:5484] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 reported cstate change: term changed from 0 to 1, leader changed from <none> to a47375810d964f128aae1e2df1370b79 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: HEALTHY } } }
W20240502 01:58:35.329703 12326 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:35.329952 12326 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:35.332743 12326 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_2'
I20240502 01:58:35.404724 11420 raft_consensus.cc:1261] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:35.405189 12310 consensus_queue.cc:1035] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:35.406131 11419 raft_consensus.cc:1261] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:35.407126 12287 consensus_queue.cc:1035] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:35.409752 11287 raft_consensus.cc:1261] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:35.410159 12287 consensus_queue.cc:1035] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 77.9369 ms
time per row: 0.0389684 ms
I20240502 01:58:35.451495 11135 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56106:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 2
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:35.456712 11267 tablet_service.cc:1467] Processing CreateTablet for tablet a408622a1ff546a6b70bb23101521b90 (DEFAULT_TABLE table=rebalance_test_table_3 [id=d37cf611838446c7a7e593d1673fa1df]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:35.456902 11531 tablet_service.cc:1467] Processing CreateTablet for tablet 055b7173c3174a019457ad258989238a (DEFAULT_TABLE table=rebalance_test_table_3 [id=d37cf611838446c7a7e593d1673fa1df]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:35.456930 11399 tablet_service.cc:1467] Processing CreateTablet for tablet a408622a1ff546a6b70bb23101521b90 (DEFAULT_TABLE table=rebalance_test_table_3 [id=d37cf611838446c7a7e593d1673fa1df]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:35.457055 11267 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a408622a1ff546a6b70bb23101521b90. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:35.457216 11399 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a408622a1ff546a6b70bb23101521b90. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:35.457216 11531 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 055b7173c3174a019457ad258989238a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:35.459358 12372 tablet_bootstrap.cc:492] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae: Bootstrap starting.
I20240502 01:58:35.459666 11267 tablet_service.cc:1467] Processing CreateTablet for tablet 055b7173c3174a019457ad258989238a (DEFAULT_TABLE table=rebalance_test_table_3 [id=d37cf611838446c7a7e593d1673fa1df]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:35.459736 11399 tablet_service.cc:1467] Processing CreateTablet for tablet e33cb78a2b63498f9c9485b8f751859e (DEFAULT_TABLE table=rebalance_test_table_3 [id=d37cf611838446c7a7e593d1673fa1df]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:35.459903 11267 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 055b7173c3174a019457ad258989238a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:35.459988 11399 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e33cb78a2b63498f9c9485b8f751859e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:35.460072 11531 tablet_service.cc:1467] Processing CreateTablet for tablet e33cb78a2b63498f9c9485b8f751859e (DEFAULT_TABLE table=rebalance_test_table_3 [id=d37cf611838446c7a7e593d1673fa1df]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:35.460306 11531 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e33cb78a2b63498f9c9485b8f751859e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:35.460705 12372 tablet_bootstrap.cc:654] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:35.461478 12373 tablet_bootstrap.cc:492] T a408622a1ff546a6b70bb23101521b90 P 6bf256be71684cdf87ef861e5890b614: Bootstrap starting.
I20240502 01:58:35.462180 12371 tablet_bootstrap.cc:492] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79: Bootstrap starting.
I20240502 01:58:35.462445 12373 tablet_bootstrap.cc:654] T a408622a1ff546a6b70bb23101521b90 P 6bf256be71684cdf87ef861e5890b614: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:35.463073 12371 tablet_bootstrap.cc:654] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:35.463289 12372 tablet_bootstrap.cc:492] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae: No bootstrap required, opened a new log
I20240502 01:58:35.463373 12372 ts_tablet_manager.cc:1367] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:58:35.463832 12372 raft_consensus.cc:348] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:35.463948 12372 raft_consensus.cc:374] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:35.463992 12372 raft_consensus.cc:729] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Initialized, Role: FOLLOWER
I20240502 01:58:35.464157 12372 consensus_queue.cc:260] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:35.464463 12372 ts_tablet_manager.cc:1398] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:35.464468 12371 tablet_bootstrap.cc:492] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79: No bootstrap required, opened a new log
I20240502 01:58:35.464553 12371 ts_tablet_manager.cc:1367] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.002s
I20240502 01:58:35.464643 12372 tablet_bootstrap.cc:492] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae: Bootstrap starting.
I20240502 01:58:35.464900 12371 raft_consensus.cc:348] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:35.464993 12371 raft_consensus.cc:374] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:35.465039 12371 raft_consensus.cc:729] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a47375810d964f128aae1e2df1370b79, State: Initialized, Role: FOLLOWER
I20240502 01:58:35.465169 12371 consensus_queue.cc:260] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:35.465448 12371 ts_tablet_manager.cc:1398] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:35.465524 12372 tablet_bootstrap.cc:654] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:35.465655 12371 tablet_bootstrap.cc:492] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79: Bootstrap starting.
I20240502 01:58:35.466485 12371 tablet_bootstrap.cc:654] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:35.466981 12373 tablet_bootstrap.cc:492] T a408622a1ff546a6b70bb23101521b90 P 6bf256be71684cdf87ef861e5890b614: No bootstrap required, opened a new log
I20240502 01:58:35.466981 12372 tablet_bootstrap.cc:492] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae: No bootstrap required, opened a new log
I20240502 01:58:35.467070 12372 ts_tablet_manager.cc:1367] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:35.467476 12372 raft_consensus.cc:348] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:35.467541 12371 tablet_bootstrap.cc:492] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79: No bootstrap required, opened a new log
I20240502 01:58:35.467587 12372 raft_consensus.cc:374] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:35.467618 12371 ts_tablet_manager.cc:1367] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 01:58:35.467631 12372 raft_consensus.cc:729] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Initialized, Role: FOLLOWER
I20240502 01:58:35.467774 12372 consensus_queue.cc:260] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:35.468067 12371 raft_consensus.cc:348] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:35.468148 12372 ts_tablet_manager.cc:1398] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:35.468170 12371 raft_consensus.cc:374] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:35.468205 12371 raft_consensus.cc:729] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a47375810d964f128aae1e2df1370b79, State: Initialized, Role: FOLLOWER
I20240502 01:58:35.468322 12371 consensus_queue.cc:260] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:35.468469 12373 ts_tablet_manager.cc:1367] T a408622a1ff546a6b70bb23101521b90 P 6bf256be71684cdf87ef861e5890b614: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.001s
I20240502 01:58:35.468590 12371 ts_tablet_manager.cc:1398] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:35.468947 12373 raft_consensus.cc:348] T a408622a1ff546a6b70bb23101521b90 P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:35.469059 12373 raft_consensus.cc:374] T a408622a1ff546a6b70bb23101521b90 P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:35.469111 12373 raft_consensus.cc:729] T a408622a1ff546a6b70bb23101521b90 P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6bf256be71684cdf87ef861e5890b614, State: Initialized, Role: FOLLOWER
I20240502 01:58:35.469256 12373 consensus_queue.cc:260] T a408622a1ff546a6b70bb23101521b90 P 6bf256be71684cdf87ef861e5890b614 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:35.469518 12373 ts_tablet_manager.cc:1398] T a408622a1ff546a6b70bb23101521b90 P 6bf256be71684cdf87ef861e5890b614: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:35.469727 12373 tablet_bootstrap.cc:492] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614: Bootstrap starting.
I20240502 01:58:35.470562 12373 tablet_bootstrap.cc:654] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:35.471890 12373 tablet_bootstrap.cc:492] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614: No bootstrap required, opened a new log
I20240502 01:58:35.471997 12373 ts_tablet_manager.cc:1367] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 01:58:35.472543 12373 raft_consensus.cc:348] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:35.472635 12373 raft_consensus.cc:374] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:35.472668 12373 raft_consensus.cc:729] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6bf256be71684cdf87ef861e5890b614, State: Initialized, Role: FOLLOWER
I20240502 01:58:35.472785 12373 consensus_queue.cc:260] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:35.472996 12373 ts_tablet_manager.cc:1398] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:35.490418 12322 raft_consensus.cc:482] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:35.490561 12322 raft_consensus.cc:504] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:35.490938 12322 leader_election.cc:288] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857)
I20240502 01:58:35.491358 11287 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "055b7173c3174a019457ad258989238a" candidate_uuid: "8c191817e0ce48b3b552af940f9d11ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a47375810d964f128aae1e2df1370b79" is_pre_election: true
I20240502 01:58:35.491509 11287 raft_consensus.cc:2454] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8c191817e0ce48b3b552af940f9d11ae in term 0.
I20240502 01:58:35.491784 11485 leader_election.cc:302] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 8c191817e0ce48b3b552af940f9d11ae, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:35.491987 12322 raft_consensus.cc:2781] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:35.492048 12322 raft_consensus.cc:482] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:35.492101 12322 raft_consensus.cc:3037] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:35.492915 12322 raft_consensus.cc:504] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:35.493192 12322 leader_election.cc:288] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 election: Requested vote from peers a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857)
I20240502 01:58:35.493503 11287 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "055b7173c3174a019457ad258989238a" candidate_uuid: "8c191817e0ce48b3b552af940f9d11ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a47375810d964f128aae1e2df1370b79"
I20240502 01:58:35.493605 11287 raft_consensus.cc:3037] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:35.494338 11287 raft_consensus.cc:2454] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8c191817e0ce48b3b552af940f9d11ae in term 1.
I20240502 01:58:35.494630 11485 leader_election.cc:302] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 8c191817e0ce48b3b552af940f9d11ae, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:35.494782 12322 raft_consensus.cc:2781] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:35.494887 12322 raft_consensus.cc:686] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Becoming Leader. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Running, Role: LEADER
I20240502 01:58:35.495028 12322 consensus_queue.cc:237] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:35.496467 11142 catalog_manager.cc:5484] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: term changed from 0 to 1, leader changed from <none> to 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67). New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } }
I20240502 01:58:35.760334 12310 raft_consensus.cc:482] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:35.760469 12310 raft_consensus.cc:504] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:35.760811 12310 leader_election.cc:288] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:35.761332 11420 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a408622a1ff546a6b70bb23101521b90" candidate_uuid: "a47375810d964f128aae1e2df1370b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf256be71684cdf87ef861e5890b614" is_pre_election: true
I20240502 01:58:35.761478 11420 raft_consensus.cc:2454] T a408622a1ff546a6b70bb23101521b90 P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a47375810d964f128aae1e2df1370b79 in term 0.
I20240502 01:58:35.761831 11221 leader_election.cc:302] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:35.762039 12310 raft_consensus.cc:2781] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:35.762102 12310 raft_consensus.cc:482] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:35.762132 12310 raft_consensus.cc:3037] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:35.851171 12310 raft_consensus.cc:504] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:35.851698 12310 leader_election.cc:288] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 election: Requested vote from peers 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:35.852319 11420 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a408622a1ff546a6b70bb23101521b90" candidate_uuid: "a47375810d964f128aae1e2df1370b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf256be71684cdf87ef861e5890b614"
I20240502 01:58:35.852463 11420 raft_consensus.cc:3037] T a408622a1ff546a6b70bb23101521b90 P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:35.853299 11420 raft_consensus.cc:2454] T a408622a1ff546a6b70bb23101521b90 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a47375810d964f128aae1e2df1370b79 in term 1.
I20240502 01:58:35.853653 11221 leader_election.cc:302] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:35.853873 12310 raft_consensus.cc:2781] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:35.853965 12310 raft_consensus.cc:686] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Becoming Leader. State: Replica: a47375810d964f128aae1e2df1370b79, State: Running, Role: LEADER
I20240502 01:58:35.854125 12310 consensus_queue.cc:237] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:35.854882 12322 raft_consensus.cc:482] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:35.855031 12322 raft_consensus.cc:504] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:35.855438 12322 leader_election.cc:288] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:35.855846 11420 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e33cb78a2b63498f9c9485b8f751859e" candidate_uuid: "8c191817e0ce48b3b552af940f9d11ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf256be71684cdf87ef861e5890b614" is_pre_election: true
I20240502 01:58:35.855958 11420 raft_consensus.cc:2454] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8c191817e0ce48b3b552af940f9d11ae in term 0.
I20240502 01:58:35.855958 11142 catalog_manager.cc:5484] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 reported cstate change: term changed from 0 to 1, leader changed from <none> to a47375810d964f128aae1e2df1370b79 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:35.856261 11486 leader_election.cc:302] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, 8c191817e0ce48b3b552af940f9d11ae; no voters:
I20240502 01:58:35.856413 12322 raft_consensus.cc:2781] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:35.856465 12322 raft_consensus.cc:482] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:35.856521 12322 raft_consensus.cc:3037] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:35.947821 12287 consensus_queue.cc:1035] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:35.950613 12322 raft_consensus.cc:504] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:35.950955 12322 leader_election.cc:288] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 election: Requested vote from peers 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:35.951346 11420 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e33cb78a2b63498f9c9485b8f751859e" candidate_uuid: "8c191817e0ce48b3b552af940f9d11ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf256be71684cdf87ef861e5890b614"
I20240502 01:58:35.951488 11420 raft_consensus.cc:3037] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:35.952574 11420 raft_consensus.cc:2454] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8c191817e0ce48b3b552af940f9d11ae in term 1.
I20240502 01:58:35.952881 11486 leader_election.cc:302] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, 8c191817e0ce48b3b552af940f9d11ae; no voters:
I20240502 01:58:35.953114 12322 raft_consensus.cc:2781] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:35.953231 12322 raft_consensus.cc:686] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Becoming Leader. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Running, Role: LEADER
I20240502 01:58:35.953375 12322 consensus_queue.cc:237] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:35.955220 11135 catalog_manager.cc:5484] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: term changed from 0 to 1, leader changed from <none> to 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67). New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } }
W20240502 01:58:36.151892 12387 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:36.152148 12387 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:36.155210 12387 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_3'
I20240502 01:58:36.249104 11420 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:36.249909 12322 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:36.251731 11420 raft_consensus.cc:1261] T a408622a1ff546a6b70bb23101521b90 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:36.254877 12310 consensus_queue.cc:1035] T a408622a1ff546a6b70bb23101521b90 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 94.7339 ms
time per row: 0.0473669 ms
I20240502 01:58:36.298031 11137 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56106:
name: "rebalance_test_table_4"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 2
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:36.304217 11267 tablet_service.cc:1467] Processing CreateTablet for tablet 1568709dd2424f188f484adba0c720d4 (DEFAULT_TABLE table=rebalance_test_table_4 [id=c6e32f615b8e4e1c937ef19c3ab2c37a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:36.304574 11267 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1568709dd2424f188f484adba0c720d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:36.304689 11266 tablet_service.cc:1467] Processing CreateTablet for tablet 495cddfb76b241a89676a723a9cc8b44 (DEFAULT_TABLE table=rebalance_test_table_4 [id=c6e32f615b8e4e1c937ef19c3ab2c37a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:36.304915 11266 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 495cddfb76b241a89676a723a9cc8b44. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:36.306805 12439 tablet_bootstrap.cc:492] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79: Bootstrap starting.
I20240502 01:58:36.306850 11399 tablet_service.cc:1467] Processing CreateTablet for tablet 1568709dd2424f188f484adba0c720d4 (DEFAULT_TABLE table=rebalance_test_table_4 [id=c6e32f615b8e4e1c937ef19c3ab2c37a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:36.307096 11399 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1568709dd2424f188f484adba0c720d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:36.307705 12439 tablet_bootstrap.cc:654] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:36.308784 12439 tablet_bootstrap.cc:492] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79: No bootstrap required, opened a new log
I20240502 01:58:36.308871 12439 ts_tablet_manager.cc:1367] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:36.309340 12439 raft_consensus.cc:348] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:36.309444 12439 raft_consensus.cc:374] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:36.309476 12439 raft_consensus.cc:729] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a47375810d964f128aae1e2df1370b79, State: Initialized, Role: FOLLOWER
I20240502 01:58:36.309620 12439 consensus_queue.cc:260] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:36.310492 11530 tablet_service.cc:1467] Processing CreateTablet for tablet 0f1917ed678a4b8798e917d9c4e511da (DEFAULT_TABLE table=rebalance_test_table_4 [id=c6e32f615b8e4e1c937ef19c3ab2c37a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:36.310534 11399 tablet_service.cc:1467] Processing CreateTablet for tablet 0f1917ed678a4b8798e917d9c4e511da (DEFAULT_TABLE table=rebalance_test_table_4 [id=c6e32f615b8e4e1c937ef19c3ab2c37a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:36.310809 11399 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f1917ed678a4b8798e917d9c4e511da. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:36.310824 11530 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f1917ed678a4b8798e917d9c4e511da. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:36.311326 12439 ts_tablet_manager.cc:1398] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:36.311592 12439 tablet_bootstrap.cc:492] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79: Bootstrap starting.
I20240502 01:58:36.312506 12439 tablet_bootstrap.cc:654] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:36.312896 12441 tablet_bootstrap.cc:492] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614: Bootstrap starting.
I20240502 01:58:36.313329 12443 tablet_bootstrap.cc:492] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae: Bootstrap starting.
I20240502 01:58:36.310492 11531 tablet_service.cc:1467] Processing CreateTablet for tablet 495cddfb76b241a89676a723a9cc8b44 (DEFAULT_TABLE table=rebalance_test_table_4 [id=c6e32f615b8e4e1c937ef19c3ab2c37a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:36.313673 11531 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 495cddfb76b241a89676a723a9cc8b44. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:36.313890 12441 tablet_bootstrap.cc:654] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:36.314277 12443 tablet_bootstrap.cc:654] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:36.315060 12441 tablet_bootstrap.cc:492] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614: No bootstrap required, opened a new log
I20240502 01:58:36.315130 12441 ts_tablet_manager.cc:1367] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:36.315433 12443 tablet_bootstrap.cc:492] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae: No bootstrap required, opened a new log
I20240502 01:58:36.315500 12443 ts_tablet_manager.cc:1367] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:36.315539 12441 raft_consensus.cc:348] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:36.315642 12441 raft_consensus.cc:374] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:36.315670 12441 raft_consensus.cc:729] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6bf256be71684cdf87ef861e5890b614, State: Initialized, Role: FOLLOWER
I20240502 01:58:36.315786 12441 consensus_queue.cc:260] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:36.315934 12443 raft_consensus.cc:348] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:36.316020 12443 raft_consensus.cc:374] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:36.316047 12443 raft_consensus.cc:729] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Initialized, Role: FOLLOWER
I20240502 01:58:36.316074 12441 ts_tablet_manager.cc:1398] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:36.316183 12443 consensus_queue.cc:260] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:36.316256 12441 tablet_bootstrap.cc:492] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614: Bootstrap starting.
I20240502 01:58:36.316872 12443 ts_tablet_manager.cc:1398] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:36.317046 12443 tablet_bootstrap.cc:492] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae: Bootstrap starting.
I20240502 01:58:36.317070 12441 tablet_bootstrap.cc:654] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:36.318068 12443 tablet_bootstrap.cc:654] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:36.319770 12441 tablet_bootstrap.cc:492] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614: No bootstrap required, opened a new log
I20240502 01:58:36.319784 12439 tablet_bootstrap.cc:492] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79: No bootstrap required, opened a new log
I20240502 01:58:36.319854 12439 ts_tablet_manager.cc:1367] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.000s
I20240502 01:58:36.319854 12441 ts_tablet_manager.cc:1367] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:58:36.320250 12441 raft_consensus.cc:348] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:36.320262 12439 raft_consensus.cc:348] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:36.320338 12441 raft_consensus.cc:374] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:36.320338 12439 raft_consensus.cc:374] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:36.320369 12441 raft_consensus.cc:729] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6bf256be71684cdf87ef861e5890b614, State: Initialized, Role: FOLLOWER
I20240502 01:58:36.320369 12439 raft_consensus.cc:729] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a47375810d964f128aae1e2df1370b79, State: Initialized, Role: FOLLOWER
I20240502 01:58:36.320482 12441 consensus_queue.cc:260] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:36.320482 12439 consensus_queue.cc:260] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:36.320727 12439 ts_tablet_manager.cc:1398] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:36.320735 12441 ts_tablet_manager.cc:1398] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:36.321468 12443 tablet_bootstrap.cc:492] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae: No bootstrap required, opened a new log
I20240502 01:58:36.321554 12443 ts_tablet_manager.cc:1367] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 01:58:36.322021 12443 raft_consensus.cc:348] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:36.322171 12443 raft_consensus.cc:374] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:36.322225 12443 raft_consensus.cc:729] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Initialized, Role: FOLLOWER
I20240502 01:58:36.322409 12443 consensus_queue.cc:260] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:36.322795 12443 ts_tablet_manager.cc:1398] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:36.586787 12188 raft_consensus.cc:482] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:36.586936 12188 raft_consensus.cc:504] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:36.587334 12188 leader_election.cc:288] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:36.587906 11552 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f1917ed678a4b8798e917d9c4e511da" candidate_uuid: "6bf256be71684cdf87ef861e5890b614" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c191817e0ce48b3b552af940f9d11ae" is_pre_election: true
I20240502 01:58:36.588049 11552 raft_consensus.cc:2454] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6bf256be71684cdf87ef861e5890b614 in term 0.
I20240502 01:58:36.588371 11354 leader_election.cc:302] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, 8c191817e0ce48b3b552af940f9d11ae; no voters:
I20240502 01:58:36.588531 12188 raft_consensus.cc:2781] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:36.588622 12188 raft_consensus.cc:482] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:36.588651 12188 raft_consensus.cc:3037] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:36.653842 12310 raft_consensus.cc:482] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:36.654008 12310 raft_consensus.cc:504] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:36.654431 12310 leader_election.cc:288] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:36.655047 11420 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1568709dd2424f188f484adba0c720d4" candidate_uuid: "a47375810d964f128aae1e2df1370b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf256be71684cdf87ef861e5890b614" is_pre_election: true
I20240502 01:58:36.655184 11420 raft_consensus.cc:2454] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a47375810d964f128aae1e2df1370b79 in term 0.
I20240502 01:58:36.655573 11221 leader_election.cc:302] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:36.655565 12188 raft_consensus.cc:504] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:36.655814 12310 raft_consensus.cc:2781] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:36.655870 12188 leader_election.cc:288] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [CANDIDATE]: Term 1 election: Requested vote from peers 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:36.655886 12310 raft_consensus.cc:482] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:36.655934 12310 raft_consensus.cc:3037] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:36.656270 11552 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f1917ed678a4b8798e917d9c4e511da" candidate_uuid: "6bf256be71684cdf87ef861e5890b614" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c191817e0ce48b3b552af940f9d11ae"
I20240502 01:58:36.656376 11552 raft_consensus.cc:3037] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:36.657193 11552 raft_consensus.cc:2454] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6bf256be71684cdf87ef861e5890b614 in term 1.
I20240502 01:58:36.657477 11354 leader_election.cc:302] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, 8c191817e0ce48b3b552af940f9d11ae; no voters:
I20240502 01:58:36.657681 12188 raft_consensus.cc:2781] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:36.657718 12310 raft_consensus.cc:504] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:36.657835 12188 raft_consensus.cc:686] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [term 1 LEADER]: Becoming Leader. State: Replica: 6bf256be71684cdf87ef861e5890b614, State: Running, Role: LEADER
I20240502 01:58:36.657975 12188 consensus_queue.cc:237] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } }
I20240502 01:58:36.658069 12310 leader_election.cc:288] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 election: Requested vote from peers 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:36.658404 11420 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1568709dd2424f188f484adba0c720d4" candidate_uuid: "a47375810d964f128aae1e2df1370b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf256be71684cdf87ef861e5890b614"
I20240502 01:58:36.658501 11420 raft_consensus.cc:3037] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:36.659430 11420 raft_consensus.cc:2454] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a47375810d964f128aae1e2df1370b79 in term 1.
I20240502 01:58:36.659701 11221 leader_election.cc:302] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:36.659868 12310 raft_consensus.cc:2781] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:36.659992 12310 raft_consensus.cc:686] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Becoming Leader. State: Replica: a47375810d964f128aae1e2df1370b79, State: Running, Role: LEADER
I20240502 01:58:36.660136 12310 consensus_queue.cc:237] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } }
I20240502 01:58:36.661101 11135 catalog_manager.cc:5484] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6bf256be71684cdf87ef861e5890b614 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "6bf256be71684cdf87ef861e5890b614" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:36.662237 11139 catalog_manager.cc:5484] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 reported cstate change: term changed from 0 to 1, leader changed from <none> to a47375810d964f128aae1e2df1370b79 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:36.676118 12383 raft_consensus.cc:482] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:36.676272 12383 raft_consensus.cc:504] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:36.676613 12383 leader_election.cc:288] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857)
I20240502 01:58:36.677084 11287 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "495cddfb76b241a89676a723a9cc8b44" candidate_uuid: "8c191817e0ce48b3b552af940f9d11ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a47375810d964f128aae1e2df1370b79" is_pre_election: true
I20240502 01:58:36.677219 11287 raft_consensus.cc:2454] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8c191817e0ce48b3b552af940f9d11ae in term 0.
I20240502 01:58:36.677531 11485 leader_election.cc:302] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 8c191817e0ce48b3b552af940f9d11ae, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:36.677721 12383 raft_consensus.cc:2781] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:36.677800 12383 raft_consensus.cc:482] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:36.677845 12383 raft_consensus.cc:3037] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:36.754215 12383 raft_consensus.cc:504] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:36.754632 12383 leader_election.cc:288] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 election: Requested vote from peers a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857)
I20240502 01:58:36.755074 11287 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "495cddfb76b241a89676a723a9cc8b44" candidate_uuid: "8c191817e0ce48b3b552af940f9d11ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a47375810d964f128aae1e2df1370b79"
I20240502 01:58:36.755228 11287 raft_consensus.cc:3037] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:36.756446 11287 raft_consensus.cc:2454] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8c191817e0ce48b3b552af940f9d11ae in term 1.
I20240502 01:58:36.756821 11485 leader_election.cc:302] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 8c191817e0ce48b3b552af940f9d11ae, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:36.757032 12383 raft_consensus.cc:2781] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:36.757139 12383 raft_consensus.cc:686] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Becoming Leader. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Running, Role: LEADER
I20240502 01:58:36.757287 12383 consensus_queue.cc:237] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } }
I20240502 01:58:36.758934 11139 catalog_manager.cc:5484] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: term changed from 0 to 1, leader changed from <none> to 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67). New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:36.973130 12452 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:36.973412 12452 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:36.976534 12452 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_4'
I20240502 01:58:37.069100 11420 raft_consensus.cc:1261] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:37.069535 12310 consensus_queue.cc:1035] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:37.071885 11287 raft_consensus.cc:1261] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:37.073531 12383 consensus_queue.cc:1035] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:37.085647 11552 raft_consensus.cc:1261] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 01:58:37.086380 12188 consensus_queue.cc:1035] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 95.2191 ms
time per row: 0.0476096 ms
I20240502 01:58:37.114780 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.68:43337
--local_ip_for_outbound_sockets=127.7.221.68
--tserver_master_addrs=127.7.221.126:32951
--webserver_port=37421
--webserver_interface=127.7.221.68
--builtin_ntp_servers=127.7.221.84:39105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 01:58:37.222240 12504 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:37.222450 12504 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:37.222517 12504 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:37.225467 12504 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:37.225549 12504 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:37.225634 12504 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.68
I20240502 01:58:37.229125 12504 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.68:43337
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.7.221.68
--webserver_port=37421
--tserver_master_addrs=127.7.221.126:32951
--never_fsync=true
--heap_profile_path=/tmp/kudu.12504
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.68
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:37.229983 12504 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:37.231184 12504 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:37.240096 12512 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:37.240748 12509 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:37.241736 12504 server_base.cc:1034] running on GCE node
W20240502 01:58:37.242372 12510 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:37.242875 12504 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:37.243477 12504 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:37.244623 12504 hybrid_clock.cc:648] HybridClock initialized: now 1714615117244603 us; error 37 us; skew 500 ppm
I20240502 01:58:37.246445 12504 webserver.cc:460] Webserver started at http://127.7.221.68:37421/ using document root <none> and password file <none>
I20240502 01:58:37.247043 12504 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:37.247120 12504 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:37.250649 12504 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.003s sys 0.001s
I20240502 01:58:37.252502 12518 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:37.253501 12504 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.002s
I20240502 01:58:37.253608 12504 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "089e23168d0b48a8baf89ee203e1b4fd"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:37.254014 12504 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:37.271370 12504 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:37.272019 12504 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:37.272173 12504 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:37.272704 12504 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:37.273619 12504 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:37.273670 12504 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:37.273733 12504 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:37.273762 12504 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:37.282287 12504 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.68:43337
I20240502 01:58:37.282334 12636 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.68:43337 every 8 connection(s)
I20240502 01:58:37.283411 12504 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 01:58:37.290335 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 12504
I20240502 01:58:37.290863 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.69:35979
--local_ip_for_outbound_sockets=127.7.221.69
--tserver_master_addrs=127.7.221.126:32951
--webserver_port=46081
--webserver_interface=127.7.221.69
--builtin_ntp_servers=127.7.221.84:39105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:37.292167 12637 heartbeater.cc:347] Connected to a master server at 127.7.221.126:32951
I20240502 01:58:37.292428 12637 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:37.292963 12637 heartbeater.cc:511] Master 127.7.221.126:32951 requested a full tablet report, sending...
I20240502 01:58:37.294159 11133 ts_manager.cc:194] Re-registered known tserver with Master: 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68:43337)
I20240502 01:58:37.295033 11133 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.68:53519
W20240502 01:58:37.401278 12640 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:37.401516 12640 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:37.401636 12640 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:37.404772 12640 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:37.404850 12640 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:37.404999 12640 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.69
I20240502 01:58:37.408926 12640 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.69:35979
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.7.221.69
--webserver_port=46081
--tserver_master_addrs=127.7.221.126:32951
--never_fsync=true
--heap_profile_path=/tmp/kudu.12640
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.69
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:37.409811 12640 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:37.410957 12640 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:37.419051 12648 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:37.419839 12646 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:37.420742 12645 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:37.421520 12640 server_base.cc:1034] running on GCE node
I20240502 01:58:37.421880 12640 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:37.422398 12640 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:37.423542 12640 hybrid_clock.cc:648] HybridClock initialized: now 1714615117423526 us; error 33 us; skew 500 ppm
I20240502 01:58:37.425310 12640 webserver.cc:460] Webserver started at http://127.7.221.69:46081/ using document root <none> and password file <none>
I20240502 01:58:37.425827 12640 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:37.425899 12640 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:37.429176 12640 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:37.430958 12654 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:37.431874 12640 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:37.431972 12640 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "0190cccef0ae48f3baa4123c0e4b49b0"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:37.432343 12640 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:37.452143 12640 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:37.452767 12640 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:37.452914 12640 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:37.453482 12640 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:37.454357 12640 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:37.454407 12640 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:37.454469 12640 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:37.454496 12640 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:37.462846 12640 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.69:35979
I20240502 01:58:37.462899 12770 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.69:35979 every 8 connection(s)
I20240502 01:58:37.464007 12640 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 01:58:37.466876 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 12640
I20240502 01:58:37.467294 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.70:46849
--local_ip_for_outbound_sockets=127.7.221.70
--tserver_master_addrs=127.7.221.126:32951
--webserver_port=36643
--webserver_interface=127.7.221.70
--builtin_ntp_servers=127.7.221.84:39105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:37.473922 12771 heartbeater.cc:347] Connected to a master server at 127.7.221.126:32951
I20240502 01:58:37.474145 12771 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:37.474718 12771 heartbeater.cc:511] Master 127.7.221.126:32951 requested a full tablet report, sending...
I20240502 01:58:37.475821 11133 ts_manager.cc:194] Re-registered known tserver with Master: 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69:35979)
I20240502 01:58:37.476675 11133 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.69:49583
W20240502 01:58:37.581326 12774 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:37.581545 12774 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:37.581588 12774 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:37.584556 12774 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:37.584614 12774 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:37.584677 12774 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.70
I20240502 01:58:37.588136 12774 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.70:46849
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.7.221.70
--webserver_port=36643
--tserver_master_addrs=127.7.221.126:32951
--never_fsync=true
--heap_profile_path=/tmp/kudu.12774
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.70
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:37.588965 12774 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:37.590106 12774 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:37.598129 12780 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:37.598896 12784 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:37.599778 12781 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:37.600500 12774 server_base.cc:1034] running on GCE node
I20240502 01:58:37.600919 12774 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:37.601516 12774 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:37.602646 12774 hybrid_clock.cc:648] HybridClock initialized: now 1714615117602625 us; error 39 us; skew 500 ppm
I20240502 01:58:37.604679 12774 webserver.cc:460] Webserver started at http://127.7.221.70:36643/ using document root <none> and password file <none>
I20240502 01:58:37.605247 12774 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:37.605334 12774 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:37.608879 12774 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.003s sys 0.000s
I20240502 01:58:37.610759 12790 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:37.611702 12774 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:37.611805 12774 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "76bcefd6842e4c09ba16b8b6aa1ef662"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:37.612236 12774 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:37.627162 12774 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:37.627743 12774 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:37.627900 12774 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:37.628436 12774 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:37.629595 12797 ts_tablet_manager.cc:516] Loading tablet metadata (0/1 complete)
I20240502 01:58:37.632669 12774 ts_tablet_manager.cc:559] Loaded tablet metadata (1 total tablets, 1 live tablets)
I20240502 01:58:37.632726 12774 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.003s user 0.000s sys 0.000s
I20240502 01:58:37.632782 12774 ts_tablet_manager.cc:574] Registering tablets (0/1 complete)
I20240502 01:58:37.634235 12774 ts_tablet_manager.cc:590] Registered 1 tablets
I20240502 01:58:37.634280 12774 ts_tablet_manager.cc:569] Time spent register tablets: real 0.002s user 0.000s sys 0.000s
I20240502 01:58:37.634514 12797 tablet_bootstrap.cc:492] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662: Bootstrap starting.
I20240502 01:58:37.644434 12774 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.70:46849
I20240502 01:58:37.644464 12906 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.70:46849 every 8 connection(s)
I20240502 01:58:37.645534 12774 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 01:58:37.648468 12797 log.cc:824] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:37.649951 12797 tablet_bootstrap.cc:492] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:37.650310 12797 tablet_bootstrap.cc:492] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662: Bootstrap complete.
I20240502 01:58:37.651506 12797 ts_tablet_manager.cc:1367] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662: Time spent bootstrapping tablet: real 0.017s user 0.011s sys 0.005s
I20240502 01:58:37.653883 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 12774
I20240502 01:58:37.654356 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.71:38753
--local_ip_for_outbound_sockets=127.7.221.71
--tserver_master_addrs=127.7.221.126:32951
--webserver_port=37341
--webserver_interface=127.7.221.71
--builtin_ntp_servers=127.7.221.84:39105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:37.654636 12797 raft_consensus.cc:348] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } }
I20240502 01:58:37.655038 12797 raft_consensus.cc:729] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 76bcefd6842e4c09ba16b8b6aa1ef662, State: Initialized, Role: FOLLOWER
I20240502 01:58:37.655575 12797 consensus_queue.cc:260] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } }
I20240502 01:58:37.656488 12797 ts_tablet_manager.cc:1398] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662: Time spent starting tablet: real 0.005s user 0.003s sys 0.002s
I20240502 01:58:37.656841 12907 heartbeater.cc:347] Connected to a master server at 127.7.221.126:32951
I20240502 01:58:37.657086 12907 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:37.657636 12907 heartbeater.cc:511] Master 127.7.221.126:32951 requested a full tablet report, sending...
I20240502 01:58:37.659467 11133 ts_manager.cc:194] Re-registered known tserver with Master: 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70:46849)
I20240502 01:58:37.660218 11133 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.70:59563
I20240502 01:58:37.661880 12907 heartbeater.cc:503] Master 127.7.221.126:32951 was elected leader, sending a full tablet report...
W20240502 01:58:37.765736 12912 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:37.765960 12912 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:37.766036 12912 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:37.768945 12912 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:37.769019 12912 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:37.769112 12912 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.71
I20240502 01:58:37.772626 12912 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.71:38753
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.7.221.71
--webserver_port=37341
--tserver_master_addrs=127.7.221.126:32951
--never_fsync=true
--heap_profile_path=/tmp/kudu.12912
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.71
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:37.773561 12912 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:37.774689 12912 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:37.782506 12920 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:37.783284 12918 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:37.784132 12917 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:37.784883 12912 server_base.cc:1034] running on GCE node
I20240502 01:58:37.785265 12912 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:37.785841 12912 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:37.787009 12912 hybrid_clock.cc:648] HybridClock initialized: now 1714615117786987 us; error 37 us; skew 500 ppm
I20240502 01:58:37.788991 12912 webserver.cc:460] Webserver started at http://127.7.221.71:37341/ using document root <none> and password file <none>
I20240502 01:58:37.789558 12912 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:37.789633 12912 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:37.792970 12912 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:37.794951 12927 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:37.795928 12912 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:37.796047 12912 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "f271c45722d545f7b33fe7f407444dcb"
format_stamp: "Formatted at 2024-05-02 01:58:29 on dist-test-slave-dmxt"
I20240502 01:58:37.796437 12912 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:37.828466 12912 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:37.829233 12912 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:37.829404 12912 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:37.829952 12912 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:37.831157 12934 ts_tablet_manager.cc:516] Loading tablet metadata (0/1 complete)
I20240502 01:58:37.834223 12912 ts_tablet_manager.cc:559] Loaded tablet metadata (1 total tablets, 1 live tablets)
I20240502 01:58:37.834285 12912 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.003s user 0.000s sys 0.000s
I20240502 01:58:37.834364 12912 ts_tablet_manager.cc:574] Registering tablets (0/1 complete)
I20240502 01:58:37.835953 12912 ts_tablet_manager.cc:590] Registered 1 tablets
I20240502 01:58:37.836000 12912 ts_tablet_manager.cc:569] Time spent register tablets: real 0.002s user 0.000s sys 0.000s
I20240502 01:58:37.836259 12934 tablet_bootstrap.cc:492] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb: Bootstrap starting.
I20240502 01:58:37.845233 12912 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.71:38753
I20240502 01:58:37.845288 13044 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.71:38753 every 8 connection(s)
I20240502 01:58:37.846228 12912 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 01:58:37.851351 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 12912
I20240502 01:58:37.851519 8053 test_util.cc:268] Using random seed: -251221121
I20240502 01:58:37.851569 12934 log.cc:824] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:37.854163 12934 tablet_bootstrap.cc:492] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb: Bootstrap replayed 1/1 log segments. Stats: ops{read=1 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 1 replicates
I20240502 01:58:37.854704 12934 tablet_bootstrap.cc:492] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb: Bootstrap complete.
I20240502 01:58:37.855877 12934 ts_tablet_manager.cc:1367] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb: Time spent bootstrapping tablet: real 0.020s user 0.006s sys 0.012s
I20240502 01:58:37.858111 13045 heartbeater.cc:347] Connected to a master server at 127.7.221.126:32951
I20240502 01:58:37.858394 13045 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:37.859246 13045 heartbeater.cc:511] Master 127.7.221.126:32951 requested a full tablet report, sending...
I20240502 01:58:37.859136 12934 raft_consensus.cc:348] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Replica starting. Triggering 1 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } }
I20240502 01:58:37.859721 12934 raft_consensus.cc:729] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: f271c45722d545f7b33fe7f407444dcb, State: Initialized, Role: FOLLOWER
I20240502 01:58:37.860205 12934 consensus_queue.cc:260] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 1.1, Last appended by leader: 1, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } }
I20240502 01:58:37.861142 12934 ts_tablet_manager.cc:1398] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb: Time spent starting tablet: real 0.005s user 0.004s sys 0.002s
I20240502 01:58:37.862321 11141 ts_manager.cc:194] Re-registered known tserver with Master: f271c45722d545f7b33fe7f407444dcb (127.7.221.71:38753)
I20240502 01:58:37.862529 8053 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_0 already exists
I20240502 01:58:37.862752 8053 test_util.cc:268] Using random seed: -251209890
I20240502 01:58:37.863235 11141 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.71:54111
I20240502 01:58:37.865365 13045 heartbeater.cc:503] Master 127.7.221.126:32951 was elected leader, sending a full tablet report...
I20240502 01:58:37.872568 8053 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_1 already exists
I20240502 01:58:37.872730 8053 test_util.cc:268] Using random seed: -251199911
I20240502 01:58:37.883601 8053 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_2 already exists
I20240502 01:58:37.883785 8053 test_util.cc:268] Using random seed: -251188857
I20240502 01:58:37.895628 8053 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_3 already exists
I20240502 01:58:37.895794 8053 test_util.cc:268] Using random seed: -251176847
W20240502 01:58:37.897183 12908 tablet.cc:2369] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 01:58:37.924701 8053 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_4 already exists
W20240502 01:58:38.097363 13046 tablet.cc:2369] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 01:58:38.296933 12637 heartbeater.cc:503] Master 127.7.221.126:32951 was elected leader, sending a full tablet report...
I20240502 01:58:38.411180 11531 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 01:58:38.414342 11266 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 01:58:38.414822 12840 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:38.424554 11399 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 01:58:38.430778 12978 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:38.437211 12568 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:38.466872 12704 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:38.478871 12771 heartbeater.cc:503] Master 127.7.221.126:32951 was elected leader, sending a full tablet report...
I20240502 01:58:38.800112 11399 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 01:58:38.818143 11531 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 01:58:38.819761 12840 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:38.820633 12704 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:38.828995 11266 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 01:58:38.839263 12568 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:38.858269 12978 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:38.917090 13230 raft_consensus.cc:482] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:38.917341 13230 raft_consensus.cc:504] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } }
I20240502 01:58:38.941296 13230 leader_election.cc:288] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers f271c45722d545f7b33fe7f407444dcb (127.7.221.71:38753)
I20240502 01:58:38.979525 12999 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1819f2f9c847491886d78211e3ca0fcc" candidate_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f271c45722d545f7b33fe7f407444dcb" is_pre_election: true
I20240502 01:58:38.979888 12999 raft_consensus.cc:2396] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 76bcefd6842e4c09ba16b8b6aa1ef662 for term 2 because replica has last-logged OpId of term: 1 index: 1, which is greater than that of the candidate, which has last-logged OpId of term: 0 index: 0.
I20240502 01:58:38.981333 12792 leader_election.cc:302] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 2 responses out of 2 voters: 1 yes votes; 1 no votes. yes voters: 76bcefd6842e4c09ba16b8b6aa1ef662; no voters: f271c45722d545f7b33fe7f407444dcb
I20240502 01:58:38.981614 13230 raft_consensus.cc:2726] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 01:58:39.015362 11552 consensus_queue.cc:237] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 39, Committed index: 39, Last appended: 1.40, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 41 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } }
I20240502 01:58:39.017120 11287 raft_consensus.cc:1261] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 40. Preceding OpId from leader: term: 1 index: 41. (index mismatch)
I20240502 01:58:39.017537 12287 consensus_queue.cc:1035] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 39, Time since last communication: 0.000s
I20240502 01:58:39.019567 12287 raft_consensus.cc:2932] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.41: config changed from index -1 to 41, NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) added. New config: { opid_index: 41 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } } }
I20240502 01:58:39.022624 11284 raft_consensus.cc:2932] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Committing config change with OpId 1.41: config changed from index -1 to 41, NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) added. New config: { opid_index: 41 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } } }
W20240502 01:58:39.035516 11485 consensus_peers.cc:485] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae -> Peer 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69:35979): Couldn't send request to peer 0190cccef0ae48f3baa4123c0e4b49b0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 495cddfb76b241a89676a723a9cc8b44. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:39.078027 11141 catalog_manager.cc:5484] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79 reported cstate change: config changed from index -1 to 41, NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 41 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } } }
I20240502 01:58:39.118845 11551 consensus_queue.cc:237] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 1.41, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 42 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } }
I20240502 01:58:39.125886 11418 raft_consensus.cc:1261] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 41. Preceding OpId from leader: term: 1 index: 42. (index mismatch)
I20240502 01:58:39.127178 12287 consensus_queue.cc:1035] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 42, Last known committed idx: 41, Time since last communication: 0.000s
W20240502 01:58:39.132556 11486 consensus_peers.cc:485] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae -> Peer 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68:43337): Couldn't send request to peer 089e23168d0b48a8baf89ee203e1b4fd. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a1596ccad20249358ffbd0a4e3a5e1cb. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:39.135181 12383 raft_consensus.cc:2932] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.42: config changed from index -1 to 42, NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) added. New config: { opid_index: 42 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } } }
I20240502 01:58:39.138152 11417 raft_consensus.cc:2932] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Committing config change with OpId 1.42: config changed from index -1 to 42, NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) added. New config: { opid_index: 42 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } } }
I20240502 01:58:39.176105 11133 catalog_manager.cc:5484] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: config changed from index -1 to 42, NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 42 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:39.200917 11551 consensus_queue.cc:237] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 46, Committed index: 46, Last appended: 1.47, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 48 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } }
W20240502 01:58:39.209870 11487 consensus_peers.cc:485] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae -> Peer 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70:46849): Couldn't send request to peer 76bcefd6842e4c09ba16b8b6aa1ef662. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2dd97f65baa14cd9b3010b5e20592f1f. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:39.212373 11417 raft_consensus.cc:1261] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 46. Preceding OpId from leader: term: 1 index: 48. (index mismatch)
I20240502 01:58:39.212894 13135 consensus_queue.cc:1035] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 48, Last known committed idx: 46, Time since last communication: 0.000s
I20240502 01:58:39.220002 11417 raft_consensus.cc:1203] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Deduplicated request from leader. Original: 1.46->[1.47-1.48] Dedup: 1.47->[1.48-1.48]
I20240502 01:58:39.227419 13248 raft_consensus.cc:2932] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.48: config changed from index -1 to 48, NON_VOTER 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) added. New config: { opid_index: 48 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } } }
I20240502 01:58:39.232969 11417 raft_consensus.cc:2932] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Committing config change with OpId 1.48: config changed from index -1 to 48, NON_VOTER 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) added. New config: { opid_index: 48 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } } }
I20240502 01:58:39.274794 11141 catalog_manager.cc:5484] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 reported cstate change: config changed from index -1 to 48, NON_VOTER 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 48 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } } }
I20240502 01:58:39.311002 11284 consensus_queue.cc:237] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 37, Committed index: 37, Last appended: 1.37, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 38 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } }
W20240502 01:58:39.316661 11220 consensus_peers.cc:485] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 -> Peer f271c45722d545f7b33fe7f407444dcb (127.7.221.71:38753): Couldn't send request to peer f271c45722d545f7b33fe7f407444dcb. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a6b4f69afc6149feac9ca1e82ae995ff. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:39.318464 11420 raft_consensus.cc:1261] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 37. Preceding OpId from leader: term: 1 index: 38. (index mismatch)
I20240502 01:58:39.319685 12185 consensus_queue.cc:1035] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 38, Last known committed idx: 37, Time since last communication: 0.000s
I20240502 01:58:39.328497 13134 raft_consensus.cc:2932] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.38: config changed from index -1 to 38, NON_VOTER f271c45722d545f7b33fe7f407444dcb (127.7.221.71) added. New config: { opid_index: 38 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } } }
I20240502 01:58:39.332778 11420 raft_consensus.cc:2932] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Committing config change with OpId 1.38: config changed from index -1 to 38, NON_VOTER f271c45722d545f7b33fe7f407444dcb (127.7.221.71) added. New config: { opid_index: 38 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } } }
I20240502 01:58:39.372388 11133 catalog_manager.cc:5484] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 reported cstate change: config changed from index -1 to 38, NON_VOTER f271c45722d545f7b33fe7f407444dcb (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 38 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:39.418980 11284 consensus_queue.cc:237] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 59, Committed index: 59, Last appended: 1.59, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 60 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } }
I20240502 01:58:39.423568 11551 raft_consensus.cc:1261] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 59. Preceding OpId from leader: term: 1 index: 60. (index mismatch)
I20240502 01:58:39.424404 13262 consensus_queue.cc:1035] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 60, Last known committed idx: 59, Time since last communication: 0.000s
I20240502 01:58:39.427093 13130 raft_consensus.cc:2932] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.60: config changed from index -1 to 60, NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) added. New config: { opid_index: 60 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } } }
I20240502 01:58:39.428628 11548 raft_consensus.cc:2932] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Committing config change with OpId 1.60: config changed from index -1 to 60, NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) added. New config: { opid_index: 60 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } } }
W20240502 01:58:39.432477 11220 consensus_peers.cc:485] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 -> Peer 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69:35979): Couldn't send request to peer 0190cccef0ae48f3baa4123c0e4b49b0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 706c5ef6b1b24ce28a70cef754b74d73. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:39.492220 11141 catalog_manager.cc:5484] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 reported cstate change: config changed from index -1 to 60, NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 60 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:39.522127 13275 ts_tablet_manager.cc:902] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0: Initiating tablet copy from peer 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:39.523095 13275 tablet_copy_client.cc:308] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44991
I20240502 01:58:39.525871 11548 consensus_queue.cc:237] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 65, Committed index: 65, Last appended: 1.65, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 66 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } }
I20240502 01:58:39.527433 11419 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 65. Preceding OpId from leader: term: 1 index: 66. (index mismatch)
I20240502 01:58:39.528034 13255 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 66, Last known committed idx: 65, Time since last communication: 0.000s
W20240502 01:58:39.530045 11486 consensus_peers.cc:485] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae -> Peer 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68:43337): Couldn't send request to peer 089e23168d0b48a8baf89ee203e1b4fd. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e33cb78a2b63498f9c9485b8f751859e. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:39.531801 13248 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.66: config changed from index -1 to 66, NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) added. New config: { opid_index: 66 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } } }
I20240502 01:58:39.532197 11419 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Committing config change with OpId 1.66: config changed from index -1 to 66, NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) added. New config: { opid_index: 66 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } } }
I20240502 01:58:39.538933 11573 tablet_copy_service.cc:140] P 8c191817e0ce48b3b552af940f9d11ae: Received BeginTabletCopySession request for tablet 495cddfb76b241a89676a723a9cc8b44 from peer 0190cccef0ae48f3baa4123c0e4b49b0 ({username='slave'} at 127.7.221.69:45885)
I20240502 01:58:39.539103 11573 tablet_copy_service.cc:161] P 8c191817e0ce48b3b552af940f9d11ae: Beginning new tablet copy session on tablet 495cddfb76b241a89676a723a9cc8b44 from peer 0190cccef0ae48f3baa4123c0e4b49b0 at {username='slave'} at 127.7.221.69:45885: session id = 0190cccef0ae48f3baa4123c0e4b49b0-495cddfb76b241a89676a723a9cc8b44
I20240502 01:58:39.540658 11573 tablet_copy_source_session.cc:205] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:39.546231 13275 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 495cddfb76b241a89676a723a9cc8b44. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:39.555688 13275 tablet_copy_client.cc:783] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:39.556142 13275 tablet_copy_client.cc:647] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:39.566900 13275 tablet_copy_client.cc:523] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:39.597445 11133 catalog_manager.cc:5484] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: config changed from index -1 to 66, NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 66 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:39.597767 13275 tablet_bootstrap.cc:492] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0: Bootstrap starting.
I20240502 01:58:39.614884 13282 raft_consensus.cc:482] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:39.615165 13282 raft_consensus.cc:504] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } }
I20240502 01:58:39.621985 13282 leader_election.cc:288] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70:46849)
I20240502 01:58:39.622843 11551 consensus_queue.cc:237] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 67, Committed index: 67, Last appended: 1.67, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } }
W20240502 01:58:39.625241 11487 consensus_peers.cc:485] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae -> Peer 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70:46849): Couldn't send request to peer 76bcefd6842e4c09ba16b8b6aa1ef662. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 055b7173c3174a019457ad258989238a. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:39.627852 11284 raft_consensus.cc:1261] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 67. Preceding OpId from leader: term: 1 index: 68. (index mismatch)
I20240502 01:58:39.628814 12383 consensus_queue.cc:1035] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 68, Last known committed idx: 67, Time since last communication: 0.000s
I20240502 01:58:39.631502 13135 raft_consensus.cc:2932] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.68: config changed from index -1 to 68, NON_VOTER 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) added. New config: { opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } } }
I20240502 01:58:39.634233 11284 raft_consensus.cc:2932] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Committing config change with OpId 1.68: config changed from index -1 to 68, NON_VOTER 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) added. New config: { opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } } }
I20240502 01:58:39.634510 13275 log.cc:824] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:39.639956 12861 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1819f2f9c847491886d78211e3ca0fcc" candidate_uuid: "f271c45722d545f7b33fe7f407444dcb" candidate_term: 2 candidate_status { last_received { term: 1 index: 1 } } ignore_live_leader: false dest_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" is_pre_election: true
I20240502 01:58:39.640277 12861 raft_consensus.cc:2454] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f271c45722d545f7b33fe7f407444dcb in term 1.
I20240502 01:58:39.641134 12931 leader_election.cc:302] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76bcefd6842e4c09ba16b8b6aa1ef662, f271c45722d545f7b33fe7f407444dcb; no voters:
I20240502 01:58:39.641363 13282 raft_consensus.cc:2781] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Leader pre-election won for term 2
I20240502 01:58:39.641436 13282 raft_consensus.cc:482] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:39.641482 13282 raft_consensus.cc:3037] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:39.653142 11133 catalog_manager.cc:5484] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: config changed from index -1 to 68, NON_VOTER 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:39.680020 11420 consensus_queue.cc:237] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 60, Committed index: 60, Last appended: 1.61, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 62 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } }
I20240502 01:58:39.688464 11551 raft_consensus.cc:1261] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 1 index: 61. Preceding OpId from leader: term: 1 index: 62. (index mismatch)
I20240502 01:58:39.688840 13254 consensus_queue.cc:1035] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 62, Last known committed idx: 60, Time since last communication: 0.000s
I20240502 01:58:39.691002 13124 raft_consensus.cc:2932] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [term 1 LEADER]: Committing config change with OpId 1.62: config changed from index -1 to 62, NON_VOTER f271c45722d545f7b33fe7f407444dcb (127.7.221.71) added. New config: { opid_index: 62 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } } }
W20240502 01:58:39.694931 11353 consensus_peers.cc:485] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 -> Peer f271c45722d545f7b33fe7f407444dcb (127.7.221.71:38753): Couldn't send request to peer f271c45722d545f7b33fe7f407444dcb. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0f1917ed678a4b8798e917d9c4e511da. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:39.695359 11551 raft_consensus.cc:2932] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Committing config change with OpId 1.62: config changed from index -1 to 62, NON_VOTER f271c45722d545f7b33fe7f407444dcb (127.7.221.71) added. New config: { opid_index: 62 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } } }
I20240502 01:58:39.698838 13282 raft_consensus.cc:504] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } }
I20240502 01:58:39.699227 13282 leader_election.cc:288] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [CANDIDATE]: Term 2 election: Requested vote from peers 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70:46849)
I20240502 01:58:39.699745 12861 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1819f2f9c847491886d78211e3ca0fcc" candidate_uuid: "f271c45722d545f7b33fe7f407444dcb" candidate_term: 2 candidate_status { last_received { term: 1 index: 1 } } ignore_live_leader: false dest_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662"
I20240502 01:58:39.699880 12861 raft_consensus.cc:3037] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:39.702168 12861 raft_consensus.cc:2454] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f271c45722d545f7b33fe7f407444dcb in term 2.
I20240502 01:58:39.702782 12931 leader_election.cc:302] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76bcefd6842e4c09ba16b8b6aa1ef662, f271c45722d545f7b33fe7f407444dcb; no voters:
I20240502 01:58:39.703014 13282 raft_consensus.cc:2781] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 2 FOLLOWER]: Leader election won for term 2
I20240502 01:58:39.703212 13282 raft_consensus.cc:686] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [term 2 LEADER]: Becoming Leader. State: Replica: f271c45722d545f7b33fe7f407444dcb, State: Running, Role: LEADER
I20240502 01:58:39.703570 13282 consensus_queue.cc:237] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 1.1, Last appended by leader: 1, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } }
I20240502 01:58:39.706776 11141 catalog_manager.cc:5484] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 reported cstate change: config changed from index -1 to 62, NON_VOTER f271c45722d545f7b33fe7f407444dcb (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "6bf256be71684cdf87ef861e5890b614" committed_config { opid_index: 62 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:39.707517 13300 ts_tablet_manager.cc:902] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd: Initiating tablet copy from peer 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:39.707880 11139 catalog_manager.cc:5484] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb reported cstate change: term changed from 1 to 2. New cstate: current_term: 2 leader_uuid: "f271c45722d545f7b33fe7f407444dcb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:39.708384 13300 tablet_copy_client.cc:308] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44991
I20240502 01:58:39.714303 13312 ts_tablet_manager.cc:902] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662: Initiating tablet copy from peer 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:39.714491 11573 tablet_copy_service.cc:140] P 8c191817e0ce48b3b552af940f9d11ae: Received BeginTabletCopySession request for tablet a1596ccad20249358ffbd0a4e3a5e1cb from peer 089e23168d0b48a8baf89ee203e1b4fd ({username='slave'} at 127.7.221.68:47765)
I20240502 01:58:39.714617 11573 tablet_copy_service.cc:161] P 8c191817e0ce48b3b552af940f9d11ae: Beginning new tablet copy session on tablet a1596ccad20249358ffbd0a4e3a5e1cb from peer 089e23168d0b48a8baf89ee203e1b4fd at {username='slave'} at 127.7.221.68:47765: session id = 089e23168d0b48a8baf89ee203e1b4fd-a1596ccad20249358ffbd0a4e3a5e1cb
I20240502 01:58:39.715077 13312 tablet_copy_client.cc:308] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44991
I20240502 01:58:39.715602 11573 tablet_copy_source_session.cc:205] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:39.719766 13300 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a1596ccad20249358ffbd0a4e3a5e1cb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:39.734174 13300 tablet_copy_client.cc:783] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:39.734380 13300 tablet_copy_client.cc:647] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:39.736636 11573 tablet_copy_service.cc:140] P 8c191817e0ce48b3b552af940f9d11ae: Received BeginTabletCopySession request for tablet 2dd97f65baa14cd9b3010b5e20592f1f from peer 76bcefd6842e4c09ba16b8b6aa1ef662 ({username='slave'} at 127.7.221.70:44693)
I20240502 01:58:39.736739 11573 tablet_copy_service.cc:161] P 8c191817e0ce48b3b552af940f9d11ae: Beginning new tablet copy session on tablet 2dd97f65baa14cd9b3010b5e20592f1f from peer 76bcefd6842e4c09ba16b8b6aa1ef662 at {username='slave'} at 127.7.221.70:44693: session id = 76bcefd6842e4c09ba16b8b6aa1ef662-2dd97f65baa14cd9b3010b5e20592f1f
I20240502 01:58:39.737596 11573 tablet_copy_source_session.cc:205] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:39.739369 13312 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2dd97f65baa14cd9b3010b5e20592f1f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:39.743785 13275 tablet_bootstrap.cc:492] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0: Bootstrap replayed 1/1 log segments. Stats: ops{read=58 overwritten=0 applied=58 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:39.744165 13275 tablet_bootstrap.cc:492] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0: Bootstrap complete.
I20240502 01:58:39.744532 13275 ts_tablet_manager.cc:1367] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0: Time spent bootstrapping tablet: real 0.147s user 0.075s sys 0.005s
I20240502 01:58:39.747280 13312 tablet_copy_client.cc:783] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:39.747473 13312 tablet_copy_client.cc:647] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:39.749143 13312 tablet_copy_client.cc:523] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:39.750447 13275 raft_consensus.cc:348] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 41 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } }
I20240502 01:58:39.750756 13275 raft_consensus.cc:729] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0190cccef0ae48f3baa4123c0e4b49b0, State: Initialized, Role: LEARNER
I20240502 01:58:39.751199 13275 consensus_queue.cc:260] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 58, Last appended: 1.58, Last appended by leader: 58, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 41 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } }
I20240502 01:58:39.752321 13275 ts_tablet_manager.cc:1398] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0: Time spent starting tablet: real 0.008s user 0.006s sys 0.000s
I20240502 01:58:39.762207 11573 tablet_copy_service.cc:342] P 8c191817e0ce48b3b552af940f9d11ae: Request end of tablet copy session 0190cccef0ae48f3baa4123c0e4b49b0-495cddfb76b241a89676a723a9cc8b44 received from {username='slave'} at 127.7.221.69:45885
I20240502 01:58:39.762331 11573 tablet_copy_service.cc:433] P 8c191817e0ce48b3b552af940f9d11ae: ending tablet copy session 0190cccef0ae48f3baa4123c0e4b49b0-495cddfb76b241a89676a723a9cc8b44 on tablet 495cddfb76b241a89676a723a9cc8b44 with peer 0190cccef0ae48f3baa4123c0e4b49b0
I20240502 01:58:39.772132 11551 consensus_queue.cc:237] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 57, Committed index: 57, Last appended: 1.57, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 58 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } }
W20240502 01:58:39.774371 11485 consensus_peers.cc:485] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae -> Peer 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69:35979): Couldn't send request to peer 0190cccef0ae48f3baa4123c0e4b49b0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4cef15d5c5fc41cbb55d184731eff9e2. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:39.775511 13300 tablet_copy_client.cc:523] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:39.781513 11285 raft_consensus.cc:1261] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 57. Preceding OpId from leader: term: 1 index: 58. (index mismatch)
I20240502 01:58:39.781879 13255 consensus_queue.cc:1035] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 58, Last known committed idx: 57, Time since last communication: 0.000s
I20240502 01:58:39.790437 13255 raft_consensus.cc:2932] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.58: config changed from index -1 to 58, NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) added. New config: { opid_index: 58 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } } }
I20240502 01:58:39.792179 11285 raft_consensus.cc:2932] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Committing config change with OpId 1.58: config changed from index -1 to 58, NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) added. New config: { opid_index: 58 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } } }
I20240502 01:58:39.794214 13312 tablet_bootstrap.cc:492] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662: Bootstrap starting.
I20240502 01:58:39.794761 13300 tablet_bootstrap.cc:492] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd: Bootstrap starting.
I20240502 01:58:39.810190 11139 catalog_manager.cc:5484] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: config changed from index -1 to 58, NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 58 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:39.820482 13300 log.cc:824] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:39.822135 11285 consensus_queue.cc:237] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 76, Committed index: 76, Last appended: 1.76, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 77 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } }
I20240502 01:58:39.825486 11419 raft_consensus.cc:1261] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 76. Preceding OpId from leader: term: 1 index: 77. (index mismatch)
I20240502 01:58:39.827507 13262 consensus_queue.cc:1035] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 77, Last known committed idx: 76, Time since last communication: 0.000s
I20240502 01:58:39.830411 13262 raft_consensus.cc:2932] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.77: config changed from index -1 to 77, NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) added. New config: { opid_index: 77 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } } }
I20240502 01:58:39.834735 11133 catalog_manager.cc:5484] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 reported cstate change: config changed from index -1 to 77, NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 77 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:39.841786 11221 consensus_peers.cc:485] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 -> Peer 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68:43337): Couldn't send request to peer 089e23168d0b48a8baf89ee203e1b4fd. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 5ddde4eecc19461db50d15139066ea31. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:39.846776 11419 raft_consensus.cc:2932] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Committing config change with OpId 1.77: config changed from index -1 to 77, NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) added. New config: { opid_index: 77 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } } }
I20240502 01:58:39.922567 13300 tablet_bootstrap.cc:492] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd: Bootstrap replayed 1/1 log segments. Stats: ops{read=59 overwritten=0 applied=59 ignored=0} inserts{seen=731 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:39.924134 13300 tablet_bootstrap.cc:492] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd: Bootstrap complete.
I20240502 01:58:39.924531 13300 ts_tablet_manager.cc:1367] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd: Time spent bootstrapping tablet: real 0.130s user 0.064s sys 0.010s
I20240502 01:58:39.936687 13335 ts_tablet_manager.cc:902] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb: Initiating tablet copy from peer a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857)
I20240502 01:58:39.937551 13335 tablet_copy_client.cc:308] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:36857
I20240502 01:58:39.938052 13300 raft_consensus.cc:348] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 42 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } }
I20240502 01:58:39.938372 13300 raft_consensus.cc:729] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 089e23168d0b48a8baf89ee203e1b4fd, State: Initialized, Role: LEARNER
I20240502 01:58:39.938828 13300 consensus_queue.cc:260] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 59, Last appended: 1.59, Last appended by leader: 59, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 42 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } }
I20240502 01:58:39.945135 11307 tablet_copy_service.cc:140] P a47375810d964f128aae1e2df1370b79: Received BeginTabletCopySession request for tablet a6b4f69afc6149feac9ca1e82ae995ff from peer f271c45722d545f7b33fe7f407444dcb ({username='slave'} at 127.7.221.71:49801)
I20240502 01:58:39.945297 11307 tablet_copy_service.cc:161] P a47375810d964f128aae1e2df1370b79: Beginning new tablet copy session on tablet a6b4f69afc6149feac9ca1e82ae995ff from peer f271c45722d545f7b33fe7f407444dcb at {username='slave'} at 127.7.221.71:49801: session id = f271c45722d545f7b33fe7f407444dcb-a6b4f69afc6149feac9ca1e82ae995ff
I20240502 01:58:39.946764 11307 tablet_copy_source_session.cc:205] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:39.945140 11284 consensus_queue.cc:237] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 68, Committed index: 68, Last appended: 1.68, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } }
I20240502 01:58:39.952713 13312 tablet_bootstrap.cc:492] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662: Bootstrap replayed 1/1 log segments. Stats: ops{read=63 overwritten=0 applied=63 ignored=0} inserts{seen=735 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:39.953085 13312 tablet_bootstrap.cc:492] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662: Bootstrap complete.
I20240502 01:58:39.953341 13312 ts_tablet_manager.cc:1367] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662: Time spent bootstrapping tablet: real 0.159s user 0.079s sys 0.000s
I20240502 01:58:39.953349 11419 raft_consensus.cc:1261] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 68. Preceding OpId from leader: term: 1 index: 69. (index mismatch)
I20240502 01:58:39.954068 13312 raft_consensus.cc:348] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 48 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } }
I20240502 01:58:39.954193 13312 raft_consensus.cc:729] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 76bcefd6842e4c09ba16b8b6aa1ef662, State: Initialized, Role: LEARNER
I20240502 01:58:39.954315 13312 consensus_queue.cc:260] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 63, Last appended: 1.63, Last appended by leader: 63, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 48 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } }
I20240502 01:58:39.954727 13312 ts_tablet_manager.cc:1398] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:39.956532 13200 consensus_queue.cc:1035] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 69, Last known committed idx: 68, Time since last communication: 0.000s
I20240502 01:58:39.956954 11573 tablet_copy_service.cc:342] P 8c191817e0ce48b3b552af940f9d11ae: Request end of tablet copy session 76bcefd6842e4c09ba16b8b6aa1ef662-2dd97f65baa14cd9b3010b5e20592f1f received from {username='slave'} at 127.7.221.70:44693
I20240502 01:58:39.957036 11573 tablet_copy_service.cc:433] P 8c191817e0ce48b3b552af940f9d11ae: ending tablet copy session 76bcefd6842e4c09ba16b8b6aa1ef662-2dd97f65baa14cd9b3010b5e20592f1f on tablet 2dd97f65baa14cd9b3010b5e20592f1f with peer 76bcefd6842e4c09ba16b8b6aa1ef662
I20240502 01:58:39.958490 13335 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6b4f69afc6149feac9ca1e82ae995ff. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:39.958827 13262 raft_consensus.cc:2932] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.69: config changed from index -1 to 69, NON_VOTER 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) added. New config: { opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } } }
I20240502 01:58:39.959301 11419 raft_consensus.cc:2932] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Committing config change with OpId 1.69: config changed from index -1 to 69, NON_VOTER 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) added. New config: { opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } } }
I20240502 01:58:39.961335 11133 catalog_manager.cc:5484] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 reported cstate change: config changed from index -1 to 69, NON_VOTER 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:39.962431 13300 ts_tablet_manager.cc:1398] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd: Time spent starting tablet: real 0.038s user 0.004s sys 0.001s
W20240502 01:58:39.963366 11222 consensus_peers.cc:485] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 -> Peer 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70:46849): Couldn't send request to peer 76bcefd6842e4c09ba16b8b6aa1ef662. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1568709dd2424f188f484adba0c720d4. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:39.965406 11573 tablet_copy_service.cc:342] P 8c191817e0ce48b3b552af940f9d11ae: Request end of tablet copy session 089e23168d0b48a8baf89ee203e1b4fd-a1596ccad20249358ffbd0a4e3a5e1cb received from {username='slave'} at 127.7.221.68:47765
I20240502 01:58:39.965497 11573 tablet_copy_service.cc:433] P 8c191817e0ce48b3b552af940f9d11ae: ending tablet copy session 089e23168d0b48a8baf89ee203e1b4fd-a1596ccad20249358ffbd0a4e3a5e1cb on tablet a1596ccad20249358ffbd0a4e3a5e1cb with peer 089e23168d0b48a8baf89ee203e1b4fd
I20240502 01:58:39.968122 13335 tablet_copy_client.cc:783] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:39.968361 13335 tablet_copy_client.cc:647] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:39.970814 12724 raft_consensus.cc:1203] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.40->[1.41-1.67] Dedup: 1.58->[1.59-1.67]
I20240502 01:58:39.974154 13275 ts_tablet_manager.cc:902] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0: Initiating tablet copy from peer a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857)
I20240502 01:58:39.974669 13275 tablet_copy_client.cc:308] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:36857
I20240502 01:58:39.976263 13335 tablet_copy_client.cc:523] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:39.977814 13335 tablet_bootstrap.cc:492] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb: Bootstrap starting.
I20240502 01:58:39.981328 11307 tablet_copy_service.cc:140] P a47375810d964f128aae1e2df1370b79: Received BeginTabletCopySession request for tablet 706c5ef6b1b24ce28a70cef754b74d73 from peer 0190cccef0ae48f3baa4123c0e4b49b0 ({username='slave'} at 127.7.221.69:43453)
I20240502 01:58:39.981420 11307 tablet_copy_service.cc:161] P a47375810d964f128aae1e2df1370b79: Beginning new tablet copy session on tablet 706c5ef6b1b24ce28a70cef754b74d73 from peer 0190cccef0ae48f3baa4123c0e4b49b0 at {username='slave'} at 127.7.221.69:43453: session id = 0190cccef0ae48f3baa4123c0e4b49b0-706c5ef6b1b24ce28a70cef754b74d73
I20240502 01:58:39.982462 11307 tablet_copy_source_session.cc:205] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:39.984930 13275 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 706c5ef6b1b24ce28a70cef754b74d73. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:39.987176 13275 tablet_copy_client.cc:783] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:39.987329 13275 tablet_copy_client.cc:647] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:39.994081 11419 consensus_queue.cc:237] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 82, Committed index: 82, Last appended: 1.82, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 83 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } }
W20240502 01:58:39.995641 11353 consensus_peers.cc:485] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 -> Peer f271c45722d545f7b33fe7f407444dcb (127.7.221.71:38753): Couldn't send request to peer f271c45722d545f7b33fe7f407444dcb. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4ba576fba5b44e149c1c6a302248fd0f. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:39.996412 11284 raft_consensus.cc:1261] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 1 index: 82. Preceding OpId from leader: term: 1 index: 83. (index mismatch)
I20240502 01:58:39.996807 13124 consensus_queue.cc:1035] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 83, Last known committed idx: 82, Time since last communication: 0.000s
I20240502 01:58:39.997045 13127 raft_consensus.cc:1050] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae: attempting to promote NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 to VOTER
I20240502 01:58:39.997501 13127 consensus_queue.cc:237] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 68, Committed index: 68, Last appended: 1.68, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } }
I20240502 01:58:39.998791 13124 raft_consensus.cc:2932] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [term 1 LEADER]: Committing config change with OpId 1.83: config changed from index -1 to 83, NON_VOTER f271c45722d545f7b33fe7f407444dcb (127.7.221.71) added. New config: { opid_index: 83 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } } }
I20240502 01:58:39.999044 11284 raft_consensus.cc:1261] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 68. Preceding OpId from leader: term: 1 index: 69. (index mismatch)
I20240502 01:58:39.999425 12287 consensus_queue.cc:1035] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 69, Last known committed idx: 68, Time since last communication: 0.000s
I20240502 01:58:40.000968 12287 raft_consensus.cc:2932] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.69: config changed from index 41 to 69, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.002671 11284 raft_consensus.cc:2932] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Committing config change with OpId 1.69: config changed from index 41 to 69, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.002911 12724 raft_consensus.cc:1261] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 68. Preceding OpId from leader: term: 1 index: 69. (index mismatch)
I20240502 01:58:40.003250 11141 catalog_manager.cc:5484] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 reported cstate change: config changed from index -1 to 83, NON_VOTER f271c45722d545f7b33fe7f407444dcb (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "6bf256be71684cdf87ef861e5890b614" committed_config { opid_index: 83 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:40.005172 11285 raft_consensus.cc:2932] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Committing config change with OpId 1.83: config changed from index -1 to 83, NON_VOTER f271c45722d545f7b33fe7f407444dcb (127.7.221.71) added. New config: { opid_index: 83 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } } }
I20240502 01:58:40.006039 11139 catalog_manager.cc:5484] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: config changed from index 41 to 69, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:40.006404 13127 consensus_queue.cc:1035] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 69, Last known committed idx: 68, Time since last communication: 0.000s
I20240502 01:58:40.007535 12724 raft_consensus.cc:2932] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 FOLLOWER]: Committing config change with OpId 1.69: config changed from index 41 to 69, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.011147 13275 tablet_copy_client.cc:523] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:40.012571 13275 tablet_bootstrap.cc:492] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0: Bootstrap starting.
I20240502 01:58:40.015061 13312 ts_tablet_manager.cc:902] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662: Initiating tablet copy from peer 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:40.015527 13312 tablet_copy_client.cc:308] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44991
I20240502 01:58:40.016112 11573 tablet_copy_service.cc:140] P 8c191817e0ce48b3b552af940f9d11ae: Received BeginTabletCopySession request for tablet 055b7173c3174a019457ad258989238a from peer 76bcefd6842e4c09ba16b8b6aa1ef662 ({username='slave'} at 127.7.221.70:44693)
I20240502 01:58:40.016213 11573 tablet_copy_service.cc:161] P 8c191817e0ce48b3b552af940f9d11ae: Beginning new tablet copy session on tablet 055b7173c3174a019457ad258989238a from peer 76bcefd6842e4c09ba16b8b6aa1ef662 at {username='slave'} at 127.7.221.70:44693: session id = 76bcefd6842e4c09ba16b8b6aa1ef662-055b7173c3174a019457ad258989238a
I20240502 01:58:40.017225 11573 tablet_copy_source_session.cc:205] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:40.018586 13312 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 055b7173c3174a019457ad258989238a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:40.020644 13312 tablet_copy_client.cc:783] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:40.020790 13312 tablet_copy_client.cc:647] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:40.030532 11551 consensus_queue.cc:237] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 69, Committed index: 69, Last appended: 1.70, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 71 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } }
I20240502 01:58:40.041276 13312 tablet_copy_client.cc:523] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:40.045766 13312 tablet_bootstrap.cc:492] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662: Bootstrap starting.
I20240502 01:58:40.052138 12723 raft_consensus.cc:1261] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 70. Preceding OpId from leader: term: 1 index: 71. (index mismatch)
I20240502 01:58:40.052969 12383 consensus_queue.cc:1035] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 71, Last known committed idx: 69, Time since last communication: 0.000s
I20240502 01:58:40.055588 12383 raft_consensus.cc:2932] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.71: config changed from index 69 to 71, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New config: { opid_index: 71 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.057487 11128 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 495cddfb76b241a89676a723a9cc8b44 with cas_config_opid_index 69: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:40.059248 11141 catalog_manager.cc:5484] T 495cddfb76b241a89676a723a9cc8b44 P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: config changed from index 69 to 71, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 71 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:40.065508 11285 consensus_queue.cc:237] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 103, Committed index: 103, Last appended: 1.103, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 104 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } }
W20240502 01:58:40.066915 11220 consensus_peers.cc:485] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 -> Peer 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69:35979): Couldn't send request to peer 0190cccef0ae48f3baa4123c0e4b49b0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9375f41e0aad43d3a9d6f5ef7ac73880. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:40.070371 12723 raft_consensus.cc:2932] T 495cddfb76b241a89676a723a9cc8b44 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 FOLLOWER]: Committing config change with OpId 1.71: config changed from index 69 to 71, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New config: { opid_index: 71 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.082238 12590 raft_consensus.cc:1203] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 1 LEARNER]: Deduplicated request from leader. Original: 1.41->[1.42-1.66] Dedup: 1.59->[1.60-1.66]
I20240502 01:58:40.082398 11551 raft_consensus.cc:1261] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 103. Preceding OpId from leader: term: 1 index: 104. (index mismatch)
I20240502 01:58:40.084908 11266 tablet_service.cc:1514] Processing DeleteTablet for tablet 495cddfb76b241a89676a723a9cc8b44 with delete_type TABLET_DATA_TOMBSTONED (TS a47375810d964f128aae1e2df1370b79 not found in new config with opid_index 71) from {username='slave'} at 127.0.0.1:49072
I20240502 01:58:40.086902 13087 consensus_queue.cc:1035] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 104, Last known committed idx: 103, Time since last communication: 0.000s
I20240502 01:58:40.098726 12861 raft_consensus.cc:1203] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.47->[1.48-1.68] Dedup: 1.63->[1.64-1.68]
I20240502 01:58:40.100462 13372 tablet_replica.cc:330] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79: stopping tablet replica
I20240502 01:58:40.100750 13372 raft_consensus.cc:2229] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:40.100894 13372 pending_rounds.cc:62] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79: Trying to abort 1 pending ops.
I20240502 01:58:40.100945 13372 pending_rounds.cc:68] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79: Aborting op as it isn't in flight: id { term: 1 index: 70 } timestamp: 7023063531622932480 op_type: WRITE_OP write_request { tablet_id: "495cddfb76b241a89676a723a9cc8b44" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000\366u\0308\016\272\305\334\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""0b19a7dd82b78960" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7023063531494395904 authz_token { token_data: "\010\371\350\313\261\006\"3\n\005slave\022*\n c6e32f615b8e4e1c937ef19c3ab2c37a\020\001\030\001 \001(\001" signature: "<redacted>""\201f\221U0\r9.\264\373\037\247\224z\224,\017\376\246\242;\257\255\021\360\n\\\025J\311\254r\360.\006m\251`\272\017\356:%rL\346\005\307\267\3521\311d\243S%\237z\267lIE\262\032" signing_key_seq_num: 0 } } request_id { client_id: "b20d5dfb58b3494c9a613b7a8eb96e3e" seq_no: 193 first_incomplete_seq_no: 193 attempt_no: 0 }
I20240502 01:58:40.101270 13372 raft_consensus.cc:2258] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:40.106137 13135 raft_consensus.cc:1050] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae: attempting to promote NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd to VOTER
I20240502 01:58:40.106484 13135 consensus_queue.cc:237] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 66, Committed index: 66, Last appended: 1.67, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } }
I20240502 01:58:40.106886 13372 ts_tablet_manager.cc:1875] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:40.108065 12590 raft_consensus.cc:1261] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 1 LEARNER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 67. Preceding OpId from leader: term: 1 index: 68. (index mismatch)
I20240502 01:58:40.108570 13255 consensus_queue.cc:1035] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 68, Last known committed idx: 66, Time since last communication: 0.000s
I20240502 01:58:40.110396 13255 raft_consensus.cc:2932] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.68: config changed from index 42 to 68, 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:40.113193 11141 catalog_manager.cc:5484] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: config changed from index 42 to 68, 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:40.113315 12590 raft_consensus.cc:2932] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Committing config change with OpId 1.68: config changed from index 42 to 68, 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:40.116690 13372 ts_tablet_manager.cc:1888] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.70
I20240502 01:58:40.118541 13300 ts_tablet_manager.cc:902] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd: Initiating tablet copy from peer 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:40.119112 13300 tablet_copy_client.cc:308] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44991
I20240502 01:58:40.119800 11573 tablet_copy_service.cc:140] P 8c191817e0ce48b3b552af940f9d11ae: Received BeginTabletCopySession request for tablet e33cb78a2b63498f9c9485b8f751859e from peer 089e23168d0b48a8baf89ee203e1b4fd ({username='slave'} at 127.7.221.68:47765)
I20240502 01:58:40.119904 11573 tablet_copy_service.cc:161] P 8c191817e0ce48b3b552af940f9d11ae: Beginning new tablet copy session on tablet e33cb78a2b63498f9c9485b8f751859e from peer 089e23168d0b48a8baf89ee203e1b4fd at {username='slave'} at 127.7.221.68:47765: session id = 089e23168d0b48a8baf89ee203e1b4fd-e33cb78a2b63498f9c9485b8f751859e
I20240502 01:58:40.120813 11573 tablet_copy_source_session.cc:205] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:40.121825 13300 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e33cb78a2b63498f9c9485b8f751859e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:40.122556 11548 consensus_queue.cc:237] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 69, Committed index: 69, Last appended: 1.69, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } }
I20240502 01:58:40.124039 13300 tablet_copy_client.cc:783] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:40.124187 13300 tablet_copy_client.cc:647] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:40.124663 12590 raft_consensus.cc:1261] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 69. Preceding OpId from leader: term: 1 index: 70. (index mismatch)
I20240502 01:58:40.125099 12383 consensus_queue.cc:1035] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 70, Last known committed idx: 69, Time since last communication: 0.000s
I20240502 01:58:40.125154 13372 log.cc:1196] T 495cddfb76b241a89676a723a9cc8b44 P a47375810d964f128aae1e2df1370b79: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/495cddfb76b241a89676a723a9cc8b44
I20240502 01:58:40.127372 11419 raft_consensus.cc:1261] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 67. Preceding OpId from leader: term: 1 index: 68. (index mismatch)
I20240502 01:58:40.127404 13135 raft_consensus.cc:2932] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.70: config changed from index 68 to 70, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New config: { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:40.128401 13275 tablet_bootstrap.cc:492] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0: Bootstrap replayed 1/1 log segments. Stats: ops{read=70 overwritten=0 applied=70 ignored=0} inserts{seen=701 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:40.128723 13275 tablet_bootstrap.cc:492] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0: Bootstrap complete.
I20240502 01:58:40.128974 13275 ts_tablet_manager.cc:1367] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0: Time spent bootstrapping tablet: real 0.116s user 0.064s sys 0.007s
I20240502 01:58:40.129086 11128 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a1596ccad20249358ffbd0a4e3a5e1cb with cas_config_opid_index 68: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:40.129581 12590 raft_consensus.cc:2932] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Committing config change with OpId 1.70: config changed from index 68 to 70, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New config: { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:40.132366 11139 catalog_manager.cc:5484] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd reported cstate change: config changed from index 68 to 70, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:40.134320 13087 raft_consensus.cc:2932] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.104: config changed from index -1 to 104, NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) added. New config: { opid_index: 104 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } } }
I20240502 01:58:40.134933 11399 tablet_service.cc:1514] Processing DeleteTablet for tablet a1596ccad20249358ffbd0a4e3a5e1cb with delete_type TABLET_DATA_TOMBSTONED (TS 6bf256be71684cdf87ef861e5890b614 not found in new config with opid_index 70) from {username='slave'} at 127.0.0.1:37868
I20240502 01:58:40.135679 13275 raft_consensus.cc:348] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 60 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } }
I20240502 01:58:40.136304 13275 raft_consensus.cc:729] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0190cccef0ae48f3baa4123c0e4b49b0, State: Initialized, Role: LEARNER
I20240502 01:58:40.136473 13275 consensus_queue.cc:260] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 70, Last appended: 1.70, Last appended by leader: 70, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 60 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } }
I20240502 01:58:40.137835 11127 catalog_manager.cc:4830] TS a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857): tablet 495cddfb76b241a89676a723a9cc8b44 (table rebalance_test_table_4 [id=c6e32f615b8e4e1c937ef19c3ab2c37a]) successfully deleted
I20240502 01:58:40.138495 13275 ts_tablet_manager.cc:1398] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0: Time spent starting tablet: real 0.009s user 0.001s sys 0.000s
I20240502 01:58:40.139688 11307 tablet_copy_service.cc:342] P a47375810d964f128aae1e2df1370b79: Request end of tablet copy session 0190cccef0ae48f3baa4123c0e4b49b0-706c5ef6b1b24ce28a70cef754b74d73 received from {username='slave'} at 127.7.221.69:43453
I20240502 01:58:40.139793 11307 tablet_copy_service.cc:433] P a47375810d964f128aae1e2df1370b79: ending tablet copy session 0190cccef0ae48f3baa4123c0e4b49b0-706c5ef6b1b24ce28a70cef754b74d73 on tablet 706c5ef6b1b24ce28a70cef754b74d73 with peer 0190cccef0ae48f3baa4123c0e4b49b0
I20240502 01:58:40.140507 11139 catalog_manager.cc:5484] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 reported cstate change: config changed from index -1 to 104, NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 104 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:40.142490 11551 raft_consensus.cc:2932] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Committing config change with OpId 1.104: config changed from index -1 to 104, NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) added. New config: { opid_index: 104 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } } }
I20240502 01:58:40.160017 13312 tablet_bootstrap.cc:492] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662: Bootstrap replayed 1/1 log segments. Stats: ops{read=78 overwritten=0 applied=78 ignored=0} inserts{seen=709 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:40.160404 13312 tablet_bootstrap.cc:492] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662: Bootstrap complete.
I20240502 01:58:40.160669 13312 ts_tablet_manager.cc:1367] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662: Time spent bootstrapping tablet: real 0.115s user 0.076s sys 0.004s
I20240502 01:58:40.161211 13312 raft_consensus.cc:348] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } }
I20240502 01:58:40.161433 13312 raft_consensus.cc:729] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 76bcefd6842e4c09ba16b8b6aa1ef662, State: Initialized, Role: LEARNER
I20240502 01:58:40.161603 13312 consensus_queue.cc:260] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 78, Last appended: 1.78, Last appended by leader: 78, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } }
I20240502 01:58:40.161947 13312 ts_tablet_manager.cc:1398] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:40.166905 13384 tablet_replica.cc:330] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614: stopping tablet replica
I20240502 01:58:40.167246 11573 tablet_copy_service.cc:342] P 8c191817e0ce48b3b552af940f9d11ae: Request end of tablet copy session 76bcefd6842e4c09ba16b8b6aa1ef662-055b7173c3174a019457ad258989238a received from {username='slave'} at 127.7.221.70:44693
I20240502 01:58:40.167336 11573 tablet_copy_service.cc:433] P 8c191817e0ce48b3b552af940f9d11ae: ending tablet copy session 76bcefd6842e4c09ba16b8b6aa1ef662-055b7173c3174a019457ad258989238a on tablet 055b7173c3174a019457ad258989238a with peer 76bcefd6842e4c09ba16b8b6aa1ef662
I20240502 01:58:40.167490 13300 tablet_copy_client.cc:523] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:40.167939 13384 raft_consensus.cc:2229] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:40.168447 13384 pending_rounds.cc:62] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614: Trying to abort 1 pending ops.
I20240502 01:58:40.168587 13384 pending_rounds.cc:68] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614: Aborting op as it isn't in flight: id { term: 1 index: 67 } timestamp: 7023063531920138240 op_type: WRITE_OP write_request { tablet_id: "a1596ccad20249358ffbd0a4e3a5e1cb" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000\236\035\274[1\272\362\250\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""e284a108e2d08d7f" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7023063531780886528 authz_token { token_data: "\010\371\350\313\261\006\"3\n\005slave\022*\n d73cd318a885440582b140642ef6635e\020\001\030\001 \001(\001" signature: "<redacted>""\306\241<\n!\361J\253.\331\246\357,;\230\010cG^\257w\006$\377\236\321\0360\271B\332\026K\237\3557\314\002X+\335\"\322U\032\200{3\320\243\332&\372X\334\311uzCk\037\014$\030" signing_key_seq_num: 0 } } request_id { client_id: "ec882a55b0144a8cb5fbe82894715672" seq_no: 211 first_incomplete_seq_no: 211 attempt_no: 0 }
I20240502 01:58:40.169066 13300 tablet_bootstrap.cc:492] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd: Bootstrap starting.
I20240502 01:58:40.169288 13384 raft_consensus.cc:2258] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:40.170796 13384 ts_tablet_manager.cc:1875] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:40.173703 13384 ts_tablet_manager.cc:1888] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.67
I20240502 01:58:40.173856 13384 log.cc:1196] T a1596ccad20249358ffbd0a4e3a5e1cb P 6bf256be71684cdf87ef861e5890b614: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/a1596ccad20249358ffbd0a4e3a5e1cb
I20240502 01:58:40.174937 11128 catalog_manager.cc:4830] TS 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035): tablet a1596ccad20249358ffbd0a4e3a5e1cb (table rebalance_test_table_0 [id=d73cd318a885440582b140642ef6635e]) successfully deleted
I20240502 01:58:40.186904 13335 tablet_bootstrap.cc:492] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb: Bootstrap replayed 1/1 log segments. Stats: ops{read=64 overwritten=0 applied=64 ignored=0} inserts{seen=749 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:40.187229 13335 tablet_bootstrap.cc:492] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb: Bootstrap complete.
I20240502 01:58:40.187420 13335 ts_tablet_manager.cc:1367] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb: Time spent bootstrapping tablet: real 0.210s user 0.073s sys 0.007s
I20240502 01:58:40.188057 13335 raft_consensus.cc:348] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 38 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } }
I20240502 01:58:40.188195 13335 raft_consensus.cc:729] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f271c45722d545f7b33fe7f407444dcb, State: Initialized, Role: LEARNER
I20240502 01:58:40.188536 13335 consensus_queue.cc:260] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 64, Last appended: 1.64, Last appended by leader: 64, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 38 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } }
I20240502 01:58:40.188855 13335 ts_tablet_manager.cc:1398] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 01:58:40.189599 11307 tablet_copy_service.cc:342] P a47375810d964f128aae1e2df1370b79: Request end of tablet copy session f271c45722d545f7b33fe7f407444dcb-a6b4f69afc6149feac9ca1e82ae995ff received from {username='slave'} at 127.7.221.71:49801
I20240502 01:58:40.189689 11307 tablet_copy_service.cc:433] P a47375810d964f128aae1e2df1370b79: ending tablet copy session f271c45722d545f7b33fe7f407444dcb-a6b4f69afc6149feac9ca1e82ae995ff on tablet a6b4f69afc6149feac9ca1e82ae995ff with peer f271c45722d545f7b33fe7f407444dcb
I20240502 01:58:40.192664 12383 raft_consensus.cc:1050] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae: attempting to promote NON_VOTER 76bcefd6842e4c09ba16b8b6aa1ef662 to VOTER
I20240502 01:58:40.193014 12383 consensus_queue.cc:237] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 69, Committed index: 69, Last appended: 1.69, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } }
I20240502 01:58:40.194418 11419 raft_consensus.cc:1261] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 69. Preceding OpId from leader: term: 1 index: 70. (index mismatch)
I20240502 01:58:40.194926 12861 raft_consensus.cc:1261] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 LEARNER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 69. Preceding OpId from leader: term: 1 index: 70. (index mismatch)
I20240502 01:58:40.195397 13135 consensus_queue.cc:1035] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 70, Last known committed idx: 69, Time since last communication: 0.000s
I20240502 01:58:40.196684 13255 consensus_queue.cc:1035] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 70, Last known committed idx: 69, Time since last communication: 0.000s
I20240502 01:58:40.198858 13135 raft_consensus.cc:2932] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.70: config changed from index 48 to 70, 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:40.201421 12861 raft_consensus.cc:2932] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Committing config change with OpId 1.70: config changed from index 48 to 70, 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:40.201939 11133 catalog_manager.cc:5484] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: config changed from index 48 to 70, 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:40.208179 11420 raft_consensus.cc:2932] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Committing config change with OpId 1.70: config changed from index 48 to 70, 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:40.229089 12861 raft_consensus.cc:1261] T 1819f2f9c847491886d78211e3ca0fcc P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 2 FOLLOWER]: Refusing update from remote peer f271c45722d545f7b33fe7f407444dcb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 2 index: 2. (index mismatch)
I20240502 01:58:40.231806 13299 consensus_queue.cc:1035] T 1819f2f9c847491886d78211e3ca0fcc P f271c45722d545f7b33fe7f407444dcb [LEADER]: Connected to new peer: Peer: permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:40.236653 13275 ts_tablet_manager.cc:902] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0: Initiating tablet copy from peer 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:40.237118 13275 tablet_copy_client.cc:308] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44991
I20240502 01:58:40.242236 11573 tablet_copy_service.cc:140] P 8c191817e0ce48b3b552af940f9d11ae: Received BeginTabletCopySession request for tablet 4cef15d5c5fc41cbb55d184731eff9e2 from peer 0190cccef0ae48f3baa4123c0e4b49b0 ({username='slave'} at 127.7.221.69:45885)
I20240502 01:58:40.242345 11573 tablet_copy_service.cc:161] P 8c191817e0ce48b3b552af940f9d11ae: Beginning new tablet copy session on tablet 4cef15d5c5fc41cbb55d184731eff9e2 from peer 0190cccef0ae48f3baa4123c0e4b49b0 at {username='slave'} at 127.7.221.69:45885: session id = 0190cccef0ae48f3baa4123c0e4b49b0-4cef15d5c5fc41cbb55d184731eff9e2
I20240502 01:58:40.243346 11573 tablet_copy_source_session.cc:205] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:40.245143 13275 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4cef15d5c5fc41cbb55d184731eff9e2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:40.247181 13275 tablet_copy_client.cc:783] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:40.247337 13275 tablet_copy_client.cc:647] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:40.258867 13335 ts_tablet_manager.cc:902] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb: Initiating tablet copy from peer 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:40.263451 13335 tablet_copy_client.cc:308] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:36035
I20240502 01:58:40.269137 11440 tablet_copy_service.cc:140] P 6bf256be71684cdf87ef861e5890b614: Received BeginTabletCopySession request for tablet 0f1917ed678a4b8798e917d9c4e511da from peer f271c45722d545f7b33fe7f407444dcb ({username='slave'} at 127.7.221.71:45253)
I20240502 01:58:40.269270 11440 tablet_copy_service.cc:161] P 6bf256be71684cdf87ef861e5890b614: Beginning new tablet copy session on tablet 0f1917ed678a4b8798e917d9c4e511da from peer f271c45722d545f7b33fe7f407444dcb at {username='slave'} at 127.7.221.71:45253: session id = f271c45722d545f7b33fe7f407444dcb-0f1917ed678a4b8798e917d9c4e511da
I20240502 01:58:40.274348 11440 tablet_copy_source_session.cc:205] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:40.276109 13335 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f1917ed678a4b8798e917d9c4e511da. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:40.278373 13335 tablet_copy_client.cc:783] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:40.278532 13335 tablet_copy_client.cc:647] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:40.300880 13300 tablet_bootstrap.cc:492] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd: Bootstrap replayed 1/1 log segments. Stats: ops{read=74 overwritten=0 applied=73 ignored=0} inserts{seen=745 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 1 replicates
I20240502 01:58:40.301254 13300 tablet_bootstrap.cc:492] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd: Bootstrap complete.
I20240502 01:58:40.301350 13275 tablet_copy_client.cc:523] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:40.301460 13300 ts_tablet_manager.cc:1367] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd: Time spent bootstrapping tablet: real 0.132s user 0.071s sys 0.007s
I20240502 01:58:40.301990 13300 raft_consensus.cc:348] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 1 LEARNER]: Replica starting. Triggering 1 pending ops. Active config: opid_index: 66 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } }
I20240502 01:58:40.302613 13300 raft_consensus.cc:729] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 089e23168d0b48a8baf89ee203e1b4fd, State: Initialized, Role: LEARNER
I20240502 01:58:40.302760 13300 consensus_queue.cc:260] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 73, Last appended: 1.74, Last appended by leader: 74, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 66 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } }
I20240502 01:58:40.304881 13300 ts_tablet_manager.cc:1398] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd: Time spent starting tablet: real 0.003s user 0.004s sys 0.000s
I20240502 01:58:40.305804 13275 tablet_bootstrap.cc:492] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0: Bootstrap starting.
I20240502 01:58:40.307757 13335 tablet_copy_client.cc:523] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:40.308144 11573 tablet_copy_service.cc:342] P 8c191817e0ce48b3b552af940f9d11ae: Request end of tablet copy session 089e23168d0b48a8baf89ee203e1b4fd-e33cb78a2b63498f9c9485b8f751859e received from {username='slave'} at 127.7.221.68:47765
I20240502 01:58:40.308218 11573 tablet_copy_service.cc:433] P 8c191817e0ce48b3b552af940f9d11ae: ending tablet copy session 089e23168d0b48a8baf89ee203e1b4fd-e33cb78a2b63498f9c9485b8f751859e on tablet e33cb78a2b63498f9c9485b8f751859e with peer 089e23168d0b48a8baf89ee203e1b4fd
I20240502 01:58:40.309273 13335 tablet_bootstrap.cc:492] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb: Bootstrap starting.
I20240502 01:58:40.377665 12999 raft_consensus.cc:1203] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 1 LEARNER]: Deduplicated request from leader. Original: 1.37->[1.38-1.77] Dedup: 1.64->[1.65-1.77]
I20240502 01:58:40.386260 13312 ts_tablet_manager.cc:902] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662: Initiating tablet copy from peer a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857)
I20240502 01:58:40.386788 13312 tablet_copy_client.cc:308] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:36857
I20240502 01:58:40.404675 13275 tablet_bootstrap.cc:492] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0: Bootstrap replayed 1/1 log segments. Stats: ops{read=68 overwritten=0 applied=68 ignored=0} inserts{seen=699 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:40.404995 13275 tablet_bootstrap.cc:492] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0: Bootstrap complete.
I20240502 01:58:40.405233 13275 ts_tablet_manager.cc:1367] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0: Time spent bootstrapping tablet: real 0.099s user 0.064s sys 0.004s
I20240502 01:58:40.405735 13275 raft_consensus.cc:348] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 58 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } }
I20240502 01:58:40.405866 13275 raft_consensus.cc:729] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0190cccef0ae48f3baa4123c0e4b49b0, State: Initialized, Role: LEARNER
I20240502 01:58:40.405982 13275 consensus_queue.cc:260] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 68, Last appended: 1.68, Last appended by leader: 68, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 58 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } }
I20240502 01:58:40.406260 13275 ts_tablet_manager.cc:1398] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0: Time spent starting tablet: real 0.001s user 0.003s sys 0.000s
I20240502 01:58:40.411509 11573 tablet_copy_service.cc:342] P 8c191817e0ce48b3b552af940f9d11ae: Request end of tablet copy session 0190cccef0ae48f3baa4123c0e4b49b0-4cef15d5c5fc41cbb55d184731eff9e2 received from {username='slave'} at 127.7.221.69:45885
I20240502 01:58:40.411595 11573 tablet_copy_service.cc:433] P 8c191817e0ce48b3b552af940f9d11ae: ending tablet copy session 0190cccef0ae48f3baa4123c0e4b49b0-4cef15d5c5fc41cbb55d184731eff9e2 on tablet 4cef15d5c5fc41cbb55d184731eff9e2 with peer 0190cccef0ae48f3baa4123c0e4b49b0
I20240502 01:58:40.415467 13300 ts_tablet_manager.cc:902] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd: Initiating tablet copy from peer a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857)
I20240502 01:58:40.415926 13300 tablet_copy_client.cc:308] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:36857
I20240502 01:58:40.417910 11307 tablet_copy_service.cc:140] P a47375810d964f128aae1e2df1370b79: Received BeginTabletCopySession request for tablet 1568709dd2424f188f484adba0c720d4 from peer 76bcefd6842e4c09ba16b8b6aa1ef662 ({username='slave'} at 127.7.221.70:50473)
I20240502 01:58:40.418002 11307 tablet_copy_service.cc:161] P a47375810d964f128aae1e2df1370b79: Beginning new tablet copy session on tablet 1568709dd2424f188f484adba0c720d4 from peer 76bcefd6842e4c09ba16b8b6aa1ef662 at {username='slave'} at 127.7.221.70:50473: session id = 76bcefd6842e4c09ba16b8b6aa1ef662-1568709dd2424f188f484adba0c720d4
I20240502 01:58:40.419184 11307 tablet_copy_source_session.cc:205] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:40.420367 13312 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1568709dd2424f188f484adba0c720d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:40.423877 13312 tablet_copy_client.cc:783] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:40.424046 13312 tablet_copy_client.cc:647] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:40.426440 13312 tablet_copy_client.cc:523] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:40.429759 13312 tablet_bootstrap.cc:492] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662: Bootstrap starting.
I20240502 01:58:40.431514 13263 raft_consensus.cc:1050] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79: attempting to promote NON_VOTER f271c45722d545f7b33fe7f407444dcb to VOTER
I20240502 01:58:40.431980 13263 consensus_queue.cc:237] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 77, Committed index: 77, Last appended: 1.77, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } }
I20240502 01:58:40.434118 11418 raft_consensus.cc:1261] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 77. Preceding OpId from leader: term: 1 index: 78. (index mismatch)
I20240502 01:58:40.434623 12999 raft_consensus.cc:1261] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 1 LEARNER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 77. Preceding OpId from leader: term: 1 index: 78. (index mismatch)
I20240502 01:58:40.435981 13262 consensus_queue.cc:1035] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 78, Last known committed idx: 77, Time since last communication: 0.000s
I20240502 01:58:40.436131 12185 consensus_queue.cc:1035] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 78, Last known committed idx: 77, Time since last communication: 0.000s
I20240502 01:58:40.436636 12723 raft_consensus.cc:1203] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.59->[1.60-1.78] Dedup: 1.70->[1.71-1.78]
I20240502 01:58:40.440016 13262 raft_consensus.cc:2932] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.78: config changed from index 38 to 78, f271c45722d545f7b33fe7f407444dcb (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:40.444356 11420 raft_consensus.cc:2932] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Committing config change with OpId 1.78: config changed from index 38 to 78, f271c45722d545f7b33fe7f407444dcb (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:40.444835 11133 catalog_manager.cc:5484] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 reported cstate change: config changed from index 38 to 78, f271c45722d545f7b33fe7f407444dcb (127.7.221.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:40.454679 11307 tablet_copy_service.cc:140] P a47375810d964f128aae1e2df1370b79: Received BeginTabletCopySession request for tablet 5ddde4eecc19461db50d15139066ea31 from peer 089e23168d0b48a8baf89ee203e1b4fd ({username='slave'} at 127.7.221.68:44673)
I20240502 01:58:40.454795 11307 tablet_copy_service.cc:161] P a47375810d964f128aae1e2df1370b79: Beginning new tablet copy session on tablet 5ddde4eecc19461db50d15139066ea31 from peer 089e23168d0b48a8baf89ee203e1b4fd at {username='slave'} at 127.7.221.68:44673: session id = 089e23168d0b48a8baf89ee203e1b4fd-5ddde4eecc19461db50d15139066ea31
I20240502 01:58:40.455691 11307 tablet_copy_source_session.cc:205] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:40.456198 12999 raft_consensus.cc:2932] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Committing config change with OpId 1.78: config changed from index 38 to 78, f271c45722d545f7b33fe7f407444dcb (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:40.459297 13300 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5ddde4eecc19461db50d15139066ea31. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:40.461504 13300 tablet_copy_client.cc:783] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:40.461678 13300 tablet_copy_client.cc:647] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:40.470049 13300 tablet_copy_client.cc:523] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:40.471839 13300 tablet_bootstrap.cc:492] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd: Bootstrap starting.
I20240502 01:58:40.494164 13087 raft_consensus.cc:1050] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79: attempting to promote NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 to VOTER
I20240502 01:58:40.499922 12861 raft_consensus.cc:1203] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.67->[1.68-1.92] Dedup: 1.78->[1.79-1.92]
I20240502 01:58:40.494529 13087 consensus_queue.cc:237] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 79, Committed index: 79, Last appended: 1.79, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 80 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } }
I20240502 01:58:40.503993 11551 raft_consensus.cc:1261] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 79. Preceding OpId from leader: term: 1 index: 80. (index mismatch)
I20240502 01:58:40.504482 12723 raft_consensus.cc:1261] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 79. Preceding OpId from leader: term: 1 index: 80. (index mismatch)
I20240502 01:58:40.504732 13134 consensus_queue.cc:1035] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 80, Last known committed idx: 79, Time since last communication: 0.000s
I20240502 01:58:40.505028 13134 consensus_queue.cc:1035] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 80, Last known committed idx: 79, Time since last communication: 0.000s
I20240502 01:58:40.506457 13362 raft_consensus.cc:2932] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.80: config changed from index 60 to 80, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 80 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.508927 11139 catalog_manager.cc:5484] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 reported cstate change: config changed from index 60 to 80, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 80 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:40.509635 11548 raft_consensus.cc:2932] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Committing config change with OpId 1.80: config changed from index 60 to 80, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 80 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.519205 12723 raft_consensus.cc:2932] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 FOLLOWER]: Committing config change with OpId 1.80: config changed from index 60 to 80, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 80 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.527020 11284 consensus_queue.cc:237] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 80, Committed index: 80, Last appended: 1.80, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 81 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } }
I20240502 01:58:40.529750 12723 raft_consensus.cc:1261] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 80. Preceding OpId from leader: term: 1 index: 81. (index mismatch)
I20240502 01:58:40.531442 13262 consensus_queue.cc:1035] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 81, Last known committed idx: 80, Time since last communication: 0.000s
I20240502 01:58:40.534082 13087 raft_consensus.cc:2932] T 706c5ef6b1b24ce28a70cef754b74d73 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.81: config changed from index 80 to 81, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New config: { opid_index: 81 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.535982 11127 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 706c5ef6b1b24ce28a70cef754b74d73 with cas_config_opid_index 80: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:40.536514 12724 raft_consensus.cc:2932] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 FOLLOWER]: Committing config change with OpId 1.81: config changed from index 80 to 81, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New config: { opid_index: 81 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.539065 11133 catalog_manager.cc:5484] T 706c5ef6b1b24ce28a70cef754b74d73 P 0190cccef0ae48f3baa4123c0e4b49b0 reported cstate change: config changed from index 80 to 81, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 81 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.544808 13472 ts_tablet_manager.cc:902] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb: Initiating tablet copy from peer 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:40.546383 13472 tablet_copy_client.cc:308] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:36035
I20240502 01:58:40.548115 11531 tablet_service.cc:1514] Processing DeleteTablet for tablet 706c5ef6b1b24ce28a70cef754b74d73 with delete_type TABLET_DATA_TOMBSTONED (TS 8c191817e0ce48b3b552af940f9d11ae not found in new config with opid_index 81) from {username='slave'} at 127.0.0.1:44872
I20240502 01:58:40.549919 11440 tablet_copy_service.cc:140] P 6bf256be71684cdf87ef861e5890b614: Received BeginTabletCopySession request for tablet 4ba576fba5b44e149c1c6a302248fd0f from peer f271c45722d545f7b33fe7f407444dcb ({username='slave'} at 127.7.221.71:45253)
I20240502 01:58:40.550037 11440 tablet_copy_service.cc:161] P 6bf256be71684cdf87ef861e5890b614: Beginning new tablet copy session on tablet 4ba576fba5b44e149c1c6a302248fd0f from peer f271c45722d545f7b33fe7f407444dcb at {username='slave'} at 127.7.221.71:45253: session id = f271c45722d545f7b33fe7f407444dcb-4ba576fba5b44e149c1c6a302248fd0f
I20240502 01:58:40.551050 11440 tablet_copy_source_session.cc:205] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:40.553315 13472 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ba576fba5b44e149c1c6a302248fd0f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:40.562533 13472 tablet_copy_client.cc:783] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:40.562873 13478 tablet_replica.cc:330] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae: stopping tablet replica
I20240502 01:58:40.563119 13472 tablet_copy_client.cc:647] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:40.563570 13478 raft_consensus.cc:2229] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:40.563923 13478 raft_consensus.cc:2258] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:40.566244 13478 ts_tablet_manager.cc:1875] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:40.567870 13127 raft_consensus.cc:1050] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae: attempting to promote NON_VOTER 76bcefd6842e4c09ba16b8b6aa1ef662 to VOTER
I20240502 01:58:40.568179 13127 consensus_queue.cc:237] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 94, Committed index: 94, Last appended: 1.95, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 96 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } }
I20240502 01:58:40.570030 12860 raft_consensus.cc:1261] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 LEARNER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 93. Preceding OpId from leader: term: 1 index: 96. (index mismatch)
I20240502 01:58:40.571107 13255 consensus_queue.cc:1035] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 96, Last known committed idx: 93, Time since last communication: 0.000s
I20240502 01:58:40.574375 11284 raft_consensus.cc:1261] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 95. Preceding OpId from leader: term: 1 index: 96. (index mismatch)
I20240502 01:58:40.575224 13472 tablet_copy_client.cc:523] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:40.576771 13472 tablet_bootstrap.cc:492] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb: Bootstrap starting.
I20240502 01:58:40.577158 12287 consensus_queue.cc:1035] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 96, Last known committed idx: 94, Time since last communication: 0.000s
I20240502 01:58:40.581842 12287 raft_consensus.cc:2932] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.96: config changed from index 68 to 96, 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 96 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:40.582006 12861 raft_consensus.cc:1203] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Deduplicated request from leader. Original: 1.93->[1.94-1.95] Dedup: 1.95->[]
I20240502 01:58:40.583240 13478 ts_tablet_manager.cc:1888] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.80
I20240502 01:58:40.583431 13478 log.cc:1196] T 706c5ef6b1b24ce28a70cef754b74d73 P 8c191817e0ce48b3b552af940f9d11ae: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/706c5ef6b1b24ce28a70cef754b74d73
I20240502 01:58:40.584820 11128 catalog_manager.cc:4830] TS 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991): tablet 706c5ef6b1b24ce28a70cef754b74d73 (table rebalance_test_table_0 [id=d73cd318a885440582b140642ef6635e]) successfully deleted
I20240502 01:58:40.585366 12861 raft_consensus.cc:2932] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Committing config change with OpId 1.96: config changed from index 68 to 96, 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 96 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:40.585587 11141 catalog_manager.cc:5484] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: config changed from index 68 to 96, 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 96 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:40.586481 13335 tablet_bootstrap.cc:492] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb: Bootstrap replayed 1/1 log segments. Stats: ops{read=71 overwritten=0 applied=71 ignored=0} inserts{seen=743 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:40.586797 13335 tablet_bootstrap.cc:492] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb: Bootstrap complete.
I20240502 01:58:40.587002 13335 ts_tablet_manager.cc:1367] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb: Time spent bootstrapping tablet: real 0.278s user 0.071s sys 0.006s
I20240502 01:58:40.587478 13335 raft_consensus.cc:348] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 62 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } }
I20240502 01:58:40.587603 13335 raft_consensus.cc:729] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f271c45722d545f7b33fe7f407444dcb, State: Initialized, Role: LEARNER
I20240502 01:58:40.587702 13335 consensus_queue.cc:260] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 71, Last appended: 1.71, Last appended by leader: 71, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 62 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } }
I20240502 01:58:40.590031 11551 consensus_queue.cc:237] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 96, Committed index: 96, Last appended: 1.96, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 97 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } }
I20240502 01:58:40.591429 12861 raft_consensus.cc:1261] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 96. Preceding OpId from leader: term: 1 index: 97. (index mismatch)
I20240502 01:58:40.592154 13127 consensus_queue.cc:1035] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 97, Last known committed idx: 96, Time since last communication: 0.000s
I20240502 01:58:40.593508 13335 ts_tablet_manager.cc:1398] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb: Time spent starting tablet: real 0.006s user 0.000s sys 0.001s
I20240502 01:58:40.594830 11440 tablet_copy_service.cc:342] P 6bf256be71684cdf87ef861e5890b614: Request end of tablet copy session f271c45722d545f7b33fe7f407444dcb-0f1917ed678a4b8798e917d9c4e511da received from {username='slave'} at 127.7.221.71:45253
I20240502 01:58:40.594949 11440 tablet_copy_service.cc:433] P 6bf256be71684cdf87ef861e5890b614: ending tablet copy session f271c45722d545f7b33fe7f407444dcb-0f1917ed678a4b8798e917d9c4e511da on tablet 0f1917ed678a4b8798e917d9c4e511da with peer f271c45722d545f7b33fe7f407444dcb
I20240502 01:58:40.595196 11285 raft_consensus.cc:2932] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Committing config change with OpId 1.96: config changed from index 68 to 96, 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 96 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:40.614053 12287 raft_consensus.cc:2932] T 055b7173c3174a019457ad258989238a P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.97: config changed from index 96 to 97, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New config: { opid_index: 97 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:40.614480 12861 raft_consensus.cc:2932] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Committing config change with OpId 1.97: config changed from index 96 to 97, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New config: { opid_index: 97 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:40.616254 11128 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 055b7173c3174a019457ad258989238a with cas_config_opid_index 96: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:40.617301 13275 ts_tablet_manager.cc:902] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0: Initiating tablet copy from peer a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857)
I20240502 01:58:40.617794 13275 tablet_copy_client.cc:308] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:36857
I20240502 01:58:40.618393 11133 catalog_manager.cc:5484] T 055b7173c3174a019457ad258989238a P 76bcefd6842e4c09ba16b8b6aa1ef662 reported cstate change: config changed from index 96 to 97, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 97 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:40.618569 11307 tablet_copy_service.cc:140] P a47375810d964f128aae1e2df1370b79: Received BeginTabletCopySession request for tablet 9375f41e0aad43d3a9d6f5ef7ac73880 from peer 0190cccef0ae48f3baa4123c0e4b49b0 ({username='slave'} at 127.7.221.69:43453)
I20240502 01:58:40.618710 11307 tablet_copy_service.cc:161] P a47375810d964f128aae1e2df1370b79: Beginning new tablet copy session on tablet 9375f41e0aad43d3a9d6f5ef7ac73880 from peer 0190cccef0ae48f3baa4123c0e4b49b0 at {username='slave'} at 127.7.221.69:43453: session id = 0190cccef0ae48f3baa4123c0e4b49b0-9375f41e0aad43d3a9d6f5ef7ac73880
I20240502 01:58:40.619685 11307 tablet_copy_source_session.cc:205] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:40.620740 13275 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9375f41e0aad43d3a9d6f5ef7ac73880. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:40.622987 13275 tablet_copy_client.cc:783] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:40.623145 13275 tablet_copy_client.cc:647] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:40.626080 11266 tablet_service.cc:1514] Processing DeleteTablet for tablet 055b7173c3174a019457ad258989238a with delete_type TABLET_DATA_TOMBSTONED (TS a47375810d964f128aae1e2df1370b79 not found in new config with opid_index 97) from {username='slave'} at 127.0.0.1:49072
I20240502 01:58:40.626116 12724 raft_consensus.cc:1203] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.57->[1.58-1.76] Dedup: 1.68->[1.69-1.76]
I20240502 01:58:40.626595 13372 tablet_replica.cc:330] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79: stopping tablet replica
I20240502 01:58:40.627303 13372 raft_consensus.cc:2229] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:40.627450 13372 raft_consensus.cc:2258] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:40.631165 13372 ts_tablet_manager.cc:1875] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:40.633389 13372 ts_tablet_manager.cc:1888] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.96
I20240502 01:58:40.633461 13372 log.cc:1196] T 055b7173c3174a019457ad258989238a P a47375810d964f128aae1e2df1370b79: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/055b7173c3174a019457ad258989238a
I20240502 01:58:40.635905 11127 catalog_manager.cc:4830] TS a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857): tablet 055b7173c3174a019457ad258989238a (table rebalance_test_table_3 [id=d37cf611838446c7a7e593d1673fa1df]) successfully deleted
I20240502 01:58:40.643008 13275 tablet_copy_client.cc:523] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:40.644490 13275 tablet_bootstrap.cc:492] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0: Bootstrap starting.
I20240502 01:58:40.656270 12590 raft_consensus.cc:1203] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 1 LEARNER]: Deduplicated request from leader. Original: 1.65->[1.66-1.89] Dedup: 1.74->[1.75-1.89]
I20240502 01:58:40.680173 13127 raft_consensus.cc:1050] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae: attempting to promote NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 to VOTER
I20240502 01:58:40.685083 13127 consensus_queue.cc:237] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 76, Committed index: 76, Last appended: 1.76, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 77 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } }
I20240502 01:58:40.689464 11284 raft_consensus.cc:1261] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 76. Preceding OpId from leader: term: 1 index: 77. (index mismatch)
I20240502 01:58:40.689464 12724 raft_consensus.cc:1261] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 76. Preceding OpId from leader: term: 1 index: 77. (index mismatch)
I20240502 01:58:40.704293 13135 consensus_queue.cc:1035] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 77, Last known committed idx: 76, Time since last communication: 0.000s
I20240502 01:58:40.707435 13135 consensus_queue.cc:1035] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 77, Last known committed idx: 76, Time since last communication: 0.000s
I20240502 01:58:40.714432 13248 raft_consensus.cc:2932] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.77: config changed from index 58 to 77, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 77 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.718515 12723 raft_consensus.cc:2932] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 FOLLOWER]: Committing config change with OpId 1.77: config changed from index 58 to 77, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 77 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.718683 11285 raft_consensus.cc:2932] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Committing config change with OpId 1.77: config changed from index 58 to 77, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 77 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.719605 13312 tablet_bootstrap.cc:492] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662: Bootstrap replayed 1/1 log segments. Stats: ops{read=76 overwritten=0 applied=75 ignored=0} inserts{seen=760 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 1 replicates
I20240502 01:58:40.719955 13312 tablet_bootstrap.cc:492] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662: Bootstrap complete.
I20240502 01:58:40.720153 13312 ts_tablet_manager.cc:1367] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662: Time spent bootstrapping tablet: real 0.290s user 0.076s sys 0.006s
I20240502 01:58:40.720657 11133 catalog_manager.cc:5484] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: config changed from index 58 to 77, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 77 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:40.722646 13312 raft_consensus.cc:348] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 LEARNER]: Replica starting. Triggering 1 pending ops. Active config: opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } }
I20240502 01:58:40.722965 13312 raft_consensus.cc:729] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 76bcefd6842e4c09ba16b8b6aa1ef662, State: Initialized, Role: LEARNER
I20240502 01:58:40.723114 13312 consensus_queue.cc:260] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 75, Last appended: 1.76, Last appended by leader: 76, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: true } }
I20240502 01:58:40.725323 13135 raft_consensus.cc:1050] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae: attempting to promote NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd to VOTER
I20240502 01:58:40.725530 13312 ts_tablet_manager.cc:1398] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 01:58:40.725647 13135 consensus_queue.cc:237] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 89, Committed index: 89, Last appended: 1.90, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 91 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } }
I20240502 01:58:40.726567 12589 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 1 LEARNER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 90. Preceding OpId from leader: term: 1 index: 91. (index mismatch)
I20240502 01:58:40.726559 11420 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 90. Preceding OpId from leader: term: 1 index: 91. (index mismatch)
I20240502 01:58:40.726974 13248 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 91, Last known committed idx: 89, Time since last communication: 0.000s
I20240502 01:58:40.727144 13255 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 91, Last known committed idx: 90, Time since last communication: 0.000s
I20240502 01:58:40.727838 11307 tablet_copy_service.cc:342] P a47375810d964f128aae1e2df1370b79: Request end of tablet copy session 76bcefd6842e4c09ba16b8b6aa1ef662-1568709dd2424f188f484adba0c720d4 received from {username='slave'} at 127.7.221.70:50473
I20240502 01:58:40.727916 11307 tablet_copy_service.cc:433] P a47375810d964f128aae1e2df1370b79: ending tablet copy session 76bcefd6842e4c09ba16b8b6aa1ef662-1568709dd2424f188f484adba0c720d4 on tablet 1568709dd2424f188f484adba0c720d4 with peer 76bcefd6842e4c09ba16b8b6aa1ef662
I20240502 01:58:40.728533 13255 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.91: config changed from index 66 to 91, 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 91 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:40.730459 11420 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Committing config change with OpId 1.91: config changed from index 66 to 91, 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 91 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:40.731092 12589 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Committing config change with OpId 1.91: config changed from index 66 to 91, 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 91 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:40.733292 11139 catalog_manager.cc:5484] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614 reported cstate change: config changed from index 66 to 91, 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 91 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:40.745054 11551 consensus_queue.cc:237] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 78, Committed index: 78, Last appended: 1.78, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 79 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } }
I20240502 01:58:40.746266 11548 consensus_queue.cc:237] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 91, Committed index: 91, Last appended: 1.91, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 92 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } }
I20240502 01:58:40.747237 12724 raft_consensus.cc:1261] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 78. Preceding OpId from leader: term: 1 index: 79. (index mismatch)
I20240502 01:58:40.749321 12287 consensus_queue.cc:1035] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 79, Last known committed idx: 78, Time since last communication: 0.000s
I20240502 01:58:40.750711 12590 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 91. Preceding OpId from leader: term: 1 index: 92. (index mismatch)
I20240502 01:58:40.751353 13255 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 92, Last known committed idx: 91, Time since last communication: 0.000s
I20240502 01:58:40.752120 13255 raft_consensus.cc:2932] T 4cef15d5c5fc41cbb55d184731eff9e2 P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.79: config changed from index 77 to 79, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New config: { opid_index: 79 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.754159 12724 raft_consensus.cc:2932] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 FOLLOWER]: Committing config change with OpId 1.79: config changed from index 77 to 79, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New config: { opid_index: 79 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.756558 11133 catalog_manager.cc:5484] T 4cef15d5c5fc41cbb55d184731eff9e2 P 0190cccef0ae48f3baa4123c0e4b49b0 reported cstate change: config changed from index 77 to 79, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 79 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:40.766906 11266 tablet_service.cc:1514] Processing DeleteTablet for tablet 4cef15d5c5fc41cbb55d184731eff9e2 with delete_type TABLET_DATA_TOMBSTONED (TS a47375810d964f128aae1e2df1370b79 not found in new config with opid_index 79) from {username='slave'} at 127.0.0.1:49072
I20240502 01:58:40.767138 13372 tablet_replica.cc:330] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79: stopping tablet replica
I20240502 01:58:40.767266 13372 raft_consensus.cc:2229] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:40.767483 13372 raft_consensus.cc:2258] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:40.768337 11128 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4cef15d5c5fc41cbb55d184731eff9e2 with cas_config_opid_index 77: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:40.769138 13372 ts_tablet_manager.cc:1875] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:40.771466 13372 ts_tablet_manager.cc:1888] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.77
I20240502 01:58:40.771593 13372 log.cc:1196] T 4cef15d5c5fc41cbb55d184731eff9e2 P a47375810d964f128aae1e2df1370b79: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/4cef15d5c5fc41cbb55d184731eff9e2
I20240502 01:58:40.772145 11127 catalog_manager.cc:4830] TS a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857): tablet 4cef15d5c5fc41cbb55d184731eff9e2 (table rebalance_test_table_2 [id=3953bb3e36b342bf9945db89810e6cf0]) successfully deleted
I20240502 01:58:40.772667 13255 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.92: config changed from index 91 to 92, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New config: { opid_index: 92 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:40.775657 12590 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Committing config change with OpId 1.92: config changed from index 91 to 92, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New config: { opid_index: 92 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:40.777406 11128 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e33cb78a2b63498f9c9485b8f751859e with cas_config_opid_index 91: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:40.777952 11133 catalog_manager.cc:5484] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: config changed from index 91 to 92, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 92 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:40.780793 11399 tablet_service.cc:1514] Processing DeleteTablet for tablet e33cb78a2b63498f9c9485b8f751859e with delete_type TABLET_DATA_TOMBSTONED (TS 6bf256be71684cdf87ef861e5890b614 not found in new config with opid_index 92) from {username='slave'} at 127.0.0.1:37868
I20240502 01:58:40.799439 12999 raft_consensus.cc:1203] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb [term 1 LEARNER]: Deduplicated request from leader. Original: 1.61->[1.62-1.77] Dedup: 1.71->[1.72-1.77]
I20240502 01:58:40.807334 13501 tablet_replica.cc:330] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614: stopping tablet replica
I20240502 01:58:40.807595 13501 raft_consensus.cc:2229] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:40.807824 13501 raft_consensus.cc:2258] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:40.813297 13501 ts_tablet_manager.cc:1875] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:40.846658 13501 ts_tablet_manager.cc:1888] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.91
I20240502 01:58:40.846884 13501 log.cc:1196] T e33cb78a2b63498f9c9485b8f751859e P 6bf256be71684cdf87ef861e5890b614: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/e33cb78a2b63498f9c9485b8f751859e
I20240502 01:58:40.850981 11128 catalog_manager.cc:4830] TS 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035): tablet e33cb78a2b63498f9c9485b8f751859e (table rebalance_test_table_3 [id=d37cf611838446c7a7e593d1673fa1df]) successfully deleted
I20240502 01:58:40.856326 13124 raft_consensus.cc:1050] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614: attempting to promote NON_VOTER f271c45722d545f7b33fe7f407444dcb to VOTER
I20240502 01:58:40.856793 13124 consensus_queue.cc:237] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 77, Committed index: 77, Last appended: 1.78, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 79 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } }
I20240502 01:58:40.859346 11548 raft_consensus.cc:1261] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 1 index: 78. Preceding OpId from leader: term: 1 index: 79. (index mismatch)
I20240502 01:58:40.859876 12999 raft_consensus.cc:1261] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb [term 1 LEARNER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 1 index: 78. Preceding OpId from leader: term: 1 index: 79. (index mismatch)
I20240502 01:58:40.860262 13172 consensus_queue.cc:1035] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 79, Last known committed idx: 77, Time since last communication: 0.000s
I20240502 01:58:40.862967 13172 raft_consensus.cc:2932] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [term 1 LEADER]: Committing config change with OpId 1.79: config changed from index 62 to 79, f271c45722d545f7b33fe7f407444dcb (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 79 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:40.863564 13124 consensus_queue.cc:1035] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 79, Last known committed idx: 77, Time since last communication: 0.000s
I20240502 01:58:40.865986 12998 raft_consensus.cc:2932] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Committing config change with OpId 1.79: config changed from index 62 to 79, f271c45722d545f7b33fe7f407444dcb (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 79 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:40.867553 11141 catalog_manager.cc:5484] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 reported cstate change: config changed from index 62 to 79, f271c45722d545f7b33fe7f407444dcb (127.7.221.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6bf256be71684cdf87ef861e5890b614" committed_config { opid_index: 79 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:40.872886 11420 consensus_queue.cc:237] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 79, Committed index: 79, Last appended: 1.80, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 81 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } }
I20240502 01:58:40.875397 11548 raft_consensus.cc:2932] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Committing config change with OpId 1.79: config changed from index 62 to 79, f271c45722d545f7b33fe7f407444dcb (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 79 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:40.876080 12997 raft_consensus.cc:1261] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 1 index: 80. Preceding OpId from leader: term: 1 index: 81. (index mismatch)
I20240502 01:58:40.877027 13113 consensus_queue.cc:1035] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 81, Last known committed idx: 79, Time since last communication: 0.000s
I20240502 01:58:40.878330 13472 tablet_bootstrap.cc:492] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb: Bootstrap replayed 1/1 log segments. Stats: ops{read=99 overwritten=0 applied=99 ignored=0} inserts{seen=784 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:40.878940 13113 raft_consensus.cc:2932] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 [term 1 LEADER]: Committing config change with OpId 1.81: config changed from index 79 to 81, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New config: { opid_index: 81 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:40.880743 11128 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0f1917ed678a4b8798e917d9c4e511da with cas_config_opid_index 79: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:40.881136 12997 raft_consensus.cc:2932] T 0f1917ed678a4b8798e917d9c4e511da P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Committing config change with OpId 1.81: config changed from index 79 to 81, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New config: { opid_index: 81 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:40.883307 11133 catalog_manager.cc:5484] T 0f1917ed678a4b8798e917d9c4e511da P 6bf256be71684cdf87ef861e5890b614 reported cstate change: config changed from index 79 to 81, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "6bf256be71684cdf87ef861e5890b614" committed_config { opid_index: 81 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:40.886718 13472 tablet_bootstrap.cc:492] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb: Bootstrap complete.
I20240502 01:58:40.886902 13472 ts_tablet_manager.cc:1367] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb: Time spent bootstrapping tablet: real 0.310s user 0.076s sys 0.022s
I20240502 01:58:40.887321 13472 raft_consensus.cc:348] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 83 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } }
I20240502 01:58:40.887410 13472 raft_consensus.cc:729] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f271c45722d545f7b33fe7f407444dcb, State: Initialized, Role: LEARNER
I20240502 01:58:40.887487 13472 consensus_queue.cc:260] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 99, Last appended: 1.99, Last appended by leader: 99, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 83 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: true } }
I20240502 01:58:40.887765 13472 ts_tablet_manager.cc:1398] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:40.890868 11531 tablet_service.cc:1514] Processing DeleteTablet for tablet 0f1917ed678a4b8798e917d9c4e511da with delete_type TABLET_DATA_TOMBSTONED (TS 8c191817e0ce48b3b552af940f9d11ae not found in new config with opid_index 81) from {username='slave'} at 127.0.0.1:44872
I20240502 01:58:40.891052 13478 tablet_replica.cc:330] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae: stopping tablet replica
I20240502 01:58:40.891165 13478 raft_consensus.cc:2229] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:40.891278 13478 raft_consensus.cc:2258] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:40.892822 13478 ts_tablet_manager.cc:1875] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:40.895023 13478 ts_tablet_manager.cc:1888] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.79
I20240502 01:58:40.895088 13478 log.cc:1196] T 0f1917ed678a4b8798e917d9c4e511da P 8c191817e0ce48b3b552af940f9d11ae: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/0f1917ed678a4b8798e917d9c4e511da
I20240502 01:58:40.895784 13275 tablet_bootstrap.cc:492] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0: Bootstrap replayed 1/1 log segments. Stats: ops{read=116 overwritten=0 applied=115 ignored=0} inserts{seen=746 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 1 replicates
I20240502 01:58:40.896134 13275 tablet_bootstrap.cc:492] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0: Bootstrap complete.
I20240502 01:58:40.896307 13275 ts_tablet_manager.cc:1367] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0: Time spent bootstrapping tablet: real 0.252s user 0.096s sys 0.009s
I20240502 01:58:40.896322 13300 tablet_bootstrap.cc:492] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd: Bootstrap replayed 1/1 log segments. Stats: ops{read=95 overwritten=0 applied=95 ignored=0} inserts{seen=780 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:40.896384 11440 tablet_copy_service.cc:342] P 6bf256be71684cdf87ef861e5890b614: Request end of tablet copy session f271c45722d545f7b33fe7f407444dcb-4ba576fba5b44e149c1c6a302248fd0f received from {username='slave'} at 127.7.221.71:45253
I20240502 01:58:40.896449 11440 tablet_copy_service.cc:433] P 6bf256be71684cdf87ef861e5890b614: ending tablet copy session f271c45722d545f7b33fe7f407444dcb-4ba576fba5b44e149c1c6a302248fd0f on tablet 4ba576fba5b44e149c1c6a302248fd0f with peer f271c45722d545f7b33fe7f407444dcb
I20240502 01:58:40.896631 13300 tablet_bootstrap.cc:492] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd: Bootstrap complete.
I20240502 01:58:40.896800 13300 ts_tablet_manager.cc:1367] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd: Time spent bootstrapping tablet: real 0.425s user 0.089s sys 0.008s
I20240502 01:58:40.897272 13300 raft_consensus.cc:348] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 77 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } }
I20240502 01:58:40.897379 13300 raft_consensus.cc:729] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 089e23168d0b48a8baf89ee203e1b4fd, State: Initialized, Role: LEARNER
I20240502 01:58:40.896837 13275 raft_consensus.cc:348] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Replica starting. Triggering 1 pending ops. Active config: opid_index: 104 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } }
I20240502 01:58:40.897468 13300 consensus_queue.cc:260] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 95, Last appended: 1.95, Last appended by leader: 95, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 77 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } }
I20240502 01:58:40.897642 13275 raft_consensus.cc:729] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0190cccef0ae48f3baa4123c0e4b49b0, State: Initialized, Role: LEARNER
I20240502 01:58:40.897769 13275 consensus_queue.cc:260] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 115, Last appended: 1.116, Last appended by leader: 116, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 104 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } }
I20240502 01:58:40.898725 13275 ts_tablet_manager.cc:1398] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 01:58:40.899339 11307 tablet_copy_service.cc:342] P a47375810d964f128aae1e2df1370b79: Request end of tablet copy session 0190cccef0ae48f3baa4123c0e4b49b0-9375f41e0aad43d3a9d6f5ef7ac73880 received from {username='slave'} at 127.7.221.69:43453
I20240502 01:58:40.899412 11307 tablet_copy_service.cc:433] P a47375810d964f128aae1e2df1370b79: ending tablet copy session 0190cccef0ae48f3baa4123c0e4b49b0-9375f41e0aad43d3a9d6f5ef7ac73880 on tablet 9375f41e0aad43d3a9d6f5ef7ac73880 with peer 0190cccef0ae48f3baa4123c0e4b49b0
I20240502 01:58:40.899905 11128 catalog_manager.cc:4830] TS 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991): tablet 0f1917ed678a4b8798e917d9c4e511da (table rebalance_test_table_4 [id=c6e32f615b8e4e1c937ef19c3ab2c37a]) successfully deleted
I20240502 01:58:40.904060 13300 ts_tablet_manager.cc:1398] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240502 01:58:40.906010 11307 tablet_copy_service.cc:342] P a47375810d964f128aae1e2df1370b79: Request end of tablet copy session 089e23168d0b48a8baf89ee203e1b4fd-5ddde4eecc19461db50d15139066ea31 received from {username='slave'} at 127.7.221.68:44673
I20240502 01:58:40.906091 11307 tablet_copy_service.cc:433] P a47375810d964f128aae1e2df1370b79: ending tablet copy session 089e23168d0b48a8baf89ee203e1b4fd-5ddde4eecc19461db50d15139066ea31 on tablet 5ddde4eecc19461db50d15139066ea31 with peer 089e23168d0b48a8baf89ee203e1b4fd
I20240502 01:58:40.944556 12999 raft_consensus.cc:1203] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb [term 1 LEARNER]: Deduplicated request from leader. Original: 1.82->[1.83-1.105] Dedup: 1.99->[1.100-1.105]
I20240502 01:58:40.978175 11548 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "2dd97f65baa14cd9b3010b5e20592f1f"
dest_uuid: "8c191817e0ce48b3b552af940f9d11ae"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:46156
I20240502 01:58:40.978349 11548 raft_consensus.cc:595] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Received request to transfer leadership
I20240502 01:58:40.994231 13135 raft_consensus.cc:979] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae: : Instructing follower 76bcefd6842e4c09ba16b8b6aa1ef662 to start an election
I20240502 01:58:40.994426 13135 raft_consensus.cc:1067] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Signalling peer 76bcefd6842e4c09ba16b8b6aa1ef662 to start an election
I20240502 01:58:40.995604 12860 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "2dd97f65baa14cd9b3010b5e20592f1f"
dest_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662"
from {username='slave'} at 127.7.221.67:42321
I20240502 01:58:40.995790 12860 raft_consensus.cc:482] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 01:58:40.995869 12860 raft_consensus.cc:3037] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:40.996960 12860 raft_consensus.cc:504] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } }
I20240502 01:58:40.997457 12860 leader_election.cc:288] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [CANDIDATE]: Term 2 election: Requested vote from peers 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991), 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:40.999074 11548 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2dd97f65baa14cd9b3010b5e20592f1f" candidate_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" candidate_term: 2 candidate_status { last_received { term: 1 index: 85 } } ignore_live_leader: true dest_uuid: "8c191817e0ce48b3b552af940f9d11ae"
I20240502 01:58:40.999198 11548 raft_consensus.cc:3032] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Stepping down as leader of term 1
I20240502 01:58:40.999234 11548 raft_consensus.cc:729] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Running, Role: LEADER
I20240502 01:58:40.999378 11548 consensus_queue.cc:260] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 85, Committed index: 85, Last appended: 1.85, Last appended by leader: 85, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } }
I20240502 01:58:40.999557 11548 raft_consensus.cc:3037] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:41.000512 11548 raft_consensus.cc:2454] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 76bcefd6842e4c09ba16b8b6aa1ef662 in term 2.
I20240502 01:58:41.001027 12793 leader_election.cc:302] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 76bcefd6842e4c09ba16b8b6aa1ef662, 8c191817e0ce48b3b552af940f9d11ae; no voters:
I20240502 01:58:41.001224 13418 raft_consensus.cc:2781] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 01:58:41.001365 13418 raft_consensus.cc:686] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 2 LEADER]: Becoming Leader. State: Replica: 76bcefd6842e4c09ba16b8b6aa1ef662, State: Running, Role: LEADER
I20240502 01:58:41.001647 13418 consensus_queue.cc:237] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 85, Committed index: 85, Last appended: 1.85, Last appended by leader: 85, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } }
I20240502 01:58:41.004458 12861 raft_consensus.cc:1203] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.68->[1.69-1.92] Dedup: 1.76->[1.77-1.92]
I20240502 01:58:41.006830 11133 catalog_manager.cc:5484] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 reported cstate change: term changed from 1 to 2, leader changed from 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) to 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70). New cstate: current_term: 2 leader_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" committed_config { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:41.008718 13113 raft_consensus.cc:1050] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614: attempting to promote NON_VOTER f271c45722d545f7b33fe7f407444dcb to VOTER
I20240502 01:58:41.009145 13113 consensus_queue.cc:237] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 105, Committed index: 105, Last appended: 1.106, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 107 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } }
I20240502 01:58:41.010977 12999 raft_consensus.cc:1261] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb [term 1 LEARNER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 1 index: 106. Preceding OpId from leader: term: 1 index: 107. (index mismatch)
I20240502 01:58:41.011803 11285 raft_consensus.cc:1261] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 1 index: 106. Preceding OpId from leader: term: 1 index: 107. (index mismatch)
I20240502 01:58:41.012295 13124 consensus_queue.cc:1035] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 107, Last known committed idx: 105, Time since last communication: 0.000s
I20240502 01:58:41.014667 13124 consensus_queue.cc:1035] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 107, Last known committed idx: 105, Time since last communication: 0.000s
I20240502 01:58:41.015897 11420 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2dd97f65baa14cd9b3010b5e20592f1f" candidate_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" candidate_term: 2 candidate_status { last_received { term: 1 index: 85 } } ignore_live_leader: true dest_uuid: "6bf256be71684cdf87ef861e5890b614"
I20240502 01:58:41.016013 11420 raft_consensus.cc:3037] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:41.016644 13254 raft_consensus.cc:2932] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [term 1 LEADER]: Committing config change with OpId 1.107: config changed from index 83 to 107, f271c45722d545f7b33fe7f407444dcb (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 107 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:41.017144 11420 raft_consensus.cc:2454] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 76bcefd6842e4c09ba16b8b6aa1ef662 in term 2.
I20240502 01:58:41.019181 11133 catalog_manager.cc:5484] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 reported cstate change: config changed from index 83 to 107, f271c45722d545f7b33fe7f407444dcb (127.7.221.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6bf256be71684cdf87ef861e5890b614" committed_config { opid_index: 107 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:41.020377 12997 raft_consensus.cc:2932] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Committing config change with OpId 1.107: config changed from index 83 to 107, f271c45722d545f7b33fe7f407444dcb (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 107 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:41.022899 11285 raft_consensus.cc:2932] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Committing config change with OpId 1.107: config changed from index 83 to 107, f271c45722d545f7b33fe7f407444dcb (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 107 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:41.031955 11420 consensus_queue.cc:237] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 107, Committed index: 107, Last appended: 1.107, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 108 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } }
I20240502 01:58:41.035553 12997 raft_consensus.cc:1261] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 1 index: 107. Preceding OpId from leader: term: 1 index: 108. (index mismatch)
I20240502 01:58:41.037159 13172 consensus_queue.cc:1035] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 108, Last known committed idx: 107, Time since last communication: 0.000s
I20240502 01:58:41.039979 13124 raft_consensus.cc:2932] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 [term 1 LEADER]: Committing config change with OpId 1.108: config changed from index 107 to 108, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New config: { opid_index: 108 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:41.040342 12997 raft_consensus.cc:2932] T 4ba576fba5b44e149c1c6a302248fd0f P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Committing config change with OpId 1.108: config changed from index 107 to 108, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New config: { opid_index: 108 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:41.042182 11128 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4ba576fba5b44e149c1c6a302248fd0f with cas_config_opid_index 107: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:41.046021 11133 catalog_manager.cc:5484] T 4ba576fba5b44e149c1c6a302248fd0f P 6bf256be71684cdf87ef861e5890b614 reported cstate change: config changed from index 107 to 108, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "6bf256be71684cdf87ef861e5890b614" committed_config { opid_index: 108 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:41.053157 13263 raft_consensus.cc:1050] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79: attempting to promote NON_VOTER 76bcefd6842e4c09ba16b8b6aa1ef662 to VOTER
I20240502 01:58:41.053526 13263 consensus_queue.cc:237] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 92, Committed index: 92, Last appended: 1.93, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 94 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } }
I20240502 01:58:41.055292 11420 raft_consensus.cc:1261] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 93. Preceding OpId from leader: term: 1 index: 94. (index mismatch)
I20240502 01:58:41.055712 13130 consensus_queue.cc:1035] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 94, Last known committed idx: 92, Time since last communication: 0.000s
I20240502 01:58:41.056411 12860 raft_consensus.cc:1261] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 LEARNER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 93. Preceding OpId from leader: term: 1 index: 94. (index mismatch)
I20240502 01:58:41.057013 13130 consensus_queue.cc:1035] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 94, Last known committed idx: 92, Time since last communication: 0.000s
I20240502 01:58:41.059000 11266 tablet_service.cc:1514] Processing DeleteTablet for tablet 4ba576fba5b44e149c1c6a302248fd0f with delete_type TABLET_DATA_TOMBSTONED (TS a47375810d964f128aae1e2df1370b79 not found in new config with opid_index 108) from {username='slave'} at 127.0.0.1:49072
I20240502 01:58:41.059221 13372 tablet_replica.cc:330] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79: stopping tablet replica
I20240502 01:58:41.059361 13372 raft_consensus.cc:2229] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:41.059494 13372 raft_consensus.cc:2258] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:41.061024 13372 ts_tablet_manager.cc:1875] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:41.061633 13087 raft_consensus.cc:2932] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.94: config changed from index 69 to 94, 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 94 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
W20240502 01:58:41.061739 11510 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46004: Illegal state: replica 8c191817e0ce48b3b552af940f9d11ae is not leader of this config: current role FOLLOWER
I20240502 01:58:41.062331 12861 raft_consensus.cc:2932] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Committing config change with OpId 1.94: config changed from index 69 to 94, 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 94 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:41.063385 13372 ts_tablet_manager.cc:1888] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.107
I20240502 01:58:41.063464 13372 log.cc:1196] T 4ba576fba5b44e149c1c6a302248fd0f P a47375810d964f128aae1e2df1370b79: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/4ba576fba5b44e149c1c6a302248fd0f
I20240502 01:58:41.065904 11127 catalog_manager.cc:4830] TS a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857): tablet 4ba576fba5b44e149c1c6a302248fd0f (table rebalance_test_table_0 [id=d73cd318a885440582b140642ef6635e]) successfully deleted
I20240502 01:58:41.065310 11133 catalog_manager.cc:5484] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 reported cstate change: config changed from index 69 to 94, 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 94 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:41.066741 11420 raft_consensus.cc:2932] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Committing config change with OpId 1.94: config changed from index 69 to 94, 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 94 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:41.071647 11285 consensus_queue.cc:237] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 94, Committed index: 94, Last appended: 1.94, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 95 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } }
I20240502 01:58:41.077840 12861 raft_consensus.cc:1261] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 94. Preceding OpId from leader: term: 1 index: 95. (index mismatch)
I20240502 01:58:41.078179 13130 consensus_queue.cc:1035] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 95, Last known committed idx: 94, Time since last communication: 0.000s
I20240502 01:58:41.079739 13263 raft_consensus.cc:2932] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.95: config changed from index 94 to 95, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New config: { opid_index: 95 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:41.081813 11127 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1568709dd2424f188f484adba0c720d4 with cas_config_opid_index 94: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:41.082077 12860 raft_consensus.cc:2932] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Committing config change with OpId 1.95: config changed from index 94 to 95, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New config: { opid_index: 95 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:41.083173 11133 catalog_manager.cc:5484] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 reported cstate change: config changed from index 94 to 95, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 95 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:41.093324 11399 tablet_service.cc:1514] Processing DeleteTablet for tablet 1568709dd2424f188f484adba0c720d4 with delete_type TABLET_DATA_TOMBSTONED (TS 6bf256be71684cdf87ef861e5890b614 not found in new config with opid_index 95) from {username='slave'} at 127.0.0.1:37868
I20240502 01:58:41.093626 13501 tablet_replica.cc:330] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614: stopping tablet replica
I20240502 01:58:41.093799 13501 raft_consensus.cc:2229] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:41.093940 13501 raft_consensus.cc:2258] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:41.095000 13501 ts_tablet_manager.cc:1875] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:41.097267 13501 ts_tablet_manager.cc:1888] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.94
I20240502 01:58:41.097342 13501 log.cc:1196] T 1568709dd2424f188f484adba0c720d4 P 6bf256be71684cdf87ef861e5890b614: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/1568709dd2424f188f484adba0c720d4
I20240502 01:58:41.097913 11128 catalog_manager.cc:4830] TS 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035): tablet 1568709dd2424f188f484adba0c720d4 (table rebalance_test_table_4 [id=c6e32f615b8e4e1c937ef19c3ab2c37a]) successfully deleted
W20240502 01:58:41.099345 11378 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:38032: Illegal state: replica 6bf256be71684cdf87ef861e5890b614 is not leader of this config: current role FOLLOWER
I20240502 01:58:41.130849 11551 raft_consensus.cc:1261] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 2 FOLLOWER]: Refusing update from remote peer 76bcefd6842e4c09ba16b8b6aa1ef662: Log matching property violated. Preceding OpId in replica: term: 1 index: 85. Preceding OpId from leader: term: 2 index: 87. (index mismatch)
I20240502 01:58:41.130846 11420 raft_consensus.cc:1261] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 2 FOLLOWER]: Refusing update from remote peer 76bcefd6842e4c09ba16b8b6aa1ef662: Log matching property violated. Preceding OpId in replica: term: 1 index: 85. Preceding OpId from leader: term: 2 index: 87. (index mismatch)
I20240502 01:58:41.131376 13418 consensus_queue.cc:1035] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 86, Last known committed idx: 85, Time since last communication: 0.000s
I20240502 01:58:41.131794 13418 consensus_queue.cc:1035] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 86, Last known committed idx: 85, Time since last communication: 0.000s
I20240502 01:58:41.132068 11285 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "a6b4f69afc6149feac9ca1e82ae995ff"
dest_uuid: "a47375810d964f128aae1e2df1370b79"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:53800
I20240502 01:58:41.132238 11285 raft_consensus.cc:595] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:58:41.136700 13470 mvcc.cc:205] Tried to move back new op lower bound from 7023063536144232448 to 7023063535625678848. Current Snapshot: MvccSnapshot[applied={T|T < 7023063536144232448 or (T in {7023063536144232448})}]
W20240502 01:58:41.152235 11246 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
I20240502 01:58:41.157114 12861 consensus_queue.cc:237] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 87, Committed index: 87, Last appended: 2.88, Last appended by leader: 85, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 89 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } }
I20240502 01:58:41.158937 11420 raft_consensus.cc:1261] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 2 FOLLOWER]: Refusing update from remote peer 76bcefd6842e4c09ba16b8b6aa1ef662: Log matching property violated. Preceding OpId in replica: term: 2 index: 87. Preceding OpId from leader: term: 2 index: 89. (index mismatch)
I20240502 01:58:41.159380 13522 consensus_queue.cc:1035] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 89, Last known committed idx: 87, Time since last communication: 0.000s
W20240502 01:58:41.162120 11246 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
I20240502 01:58:41.162909 13522 raft_consensus.cc:2932] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 2 LEADER]: Committing config change with OpId 2.89: config changed from index 70 to 89, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New config: { opid_index: 89 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:41.164978 11129 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2dd97f65baa14cd9b3010b5e20592f1f with cas_config_opid_index 70: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:41.166252 11418 raft_consensus.cc:2932] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 2 FOLLOWER]: Committing config change with OpId 2.89: config changed from index 70 to 89, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New config: { opid_index: 89 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } }
I20240502 01:58:41.169309 11141 catalog_manager.cc:5484] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 reported cstate change: config changed from index 70 to 89, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New cstate: current_term: 2 leader_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" committed_config { opid_index: 89 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
W20240502 01:58:41.171563 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
I20240502 01:58:41.173980 11531 tablet_service.cc:1514] Processing DeleteTablet for tablet 2dd97f65baa14cd9b3010b5e20592f1f with delete_type TABLET_DATA_TOMBSTONED (TS 8c191817e0ce48b3b552af940f9d11ae not found in new config with opid_index 89) from {username='slave'} at 127.0.0.1:44872
I20240502 01:58:41.174197 13478 tablet_replica.cc:330] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae: stopping tablet replica
I20240502 01:58:41.174346 13478 raft_consensus.cc:2229] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:41.174446 13478 pending_rounds.cc:62] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae: Trying to abort 1 pending ops.
I20240502 01:58:41.174516 13478 pending_rounds.cc:68] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae: Aborting op as it isn't in flight: id { term: 2 index: 88 } timestamp: 7023063536255332352 op_type: WRITE_OP write_request { tablet_id: "2dd97f65baa14cd9b3010b5e20592f1f" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000-\373\034-\374\307\275V\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""5004a13bbfeaa30f" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7023063536233865216 authz_token { token_data: "\010\371\350\313\261\006\"3\n\005slave\022*\n 3953bb3e36b342bf9945db89810e6cf0\020\001\030\001 \001(\001" signature: "<redacted>""\307n\312\220\253#\327\327e\316\241\030\276\310m\230\370\231\340D\347\273A]\337\360\321\340\327\300\234\275\352\253OK8M7\013\370\'\353\235V-\217`\373K\242}*\360\343\237+bO\356$\013Mm" signing_key_seq_num: 0 } } request_id { client_id: "97ccfc0f71f64ea89955d2414c78dff6" seq_no: 278 first_incomplete_seq_no: 278 attempt_no: 0 }
I20240502 01:58:41.174870 13478 raft_consensus.cc:2258] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:41.176226 13478 ts_tablet_manager.cc:1875] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
W20240502 01:58:41.179447 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
I20240502 01:58:41.182528 13478 ts_tablet_manager.cc:1888] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.88
I20240502 01:58:41.183496 13478 log.cc:1196] T 2dd97f65baa14cd9b3010b5e20592f1f P 8c191817e0ce48b3b552af940f9d11ae: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/2dd97f65baa14cd9b3010b5e20592f1f
I20240502 01:58:41.186383 11128 catalog_manager.cc:4830] TS 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991): tablet 2dd97f65baa14cd9b3010b5e20592f1f (table rebalance_test_table_2 [id=3953bb3e36b342bf9945db89810e6cf0]) successfully deleted
W20240502 01:58:41.189510 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
W20240502 01:58:41.198019 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
W20240502 01:58:41.210001 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
I20240502 01:58:41.212417 12724 raft_consensus.cc:1203] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.103->[1.104-1.136] Dedup: 1.116->[1.117-1.136]
W20240502 01:58:41.223018 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
W20240502 01:58:41.234299 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
W20240502 01:58:41.248410 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
W20240502 01:58:41.263192 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
W20240502 01:58:41.283468 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
W20240502 01:58:41.299913 11246 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
W20240502 01:58:41.319332 11246 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
W20240502 01:58:41.338515 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
W20240502 01:58:41.358944 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
I20240502 01:58:41.370222 11285 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "a6b4f69afc6149feac9ca1e82ae995ff"
dest_uuid: "a47375810d964f128aae1e2df1370b79"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:53852
I20240502 01:58:41.370388 11285 raft_consensus.cc:595] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Received request to transfer leadership
W20240502 01:58:41.380079 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
W20240502 01:58:41.402983 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
W20240502 01:58:41.425931 11246 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
I20240502 01:58:41.426674 11285 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "a6b4f69afc6149feac9ca1e82ae995ff"
dest_uuid: "a47375810d964f128aae1e2df1370b79"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:53896
I20240502 01:58:41.426800 11285 raft_consensus.cc:595] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:58:41.432456 12590 raft_consensus.cc:1203] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd [term 1 LEARNER]: Deduplicated request from leader. Original: 1.76->[1.77-1.128] Dedup: 1.95->[1.96-1.128]
I20240502 01:58:41.450903 13263 raft_consensus.cc:1050] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79: attempting to promote NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd to VOTER
I20240502 01:58:41.451341 13263 consensus_queue.cc:237] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 130, Committed index: 130, Last appended: 1.130, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 131 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } }
W20240502 01:58:41.452039 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
I20240502 01:58:41.452941 11420 raft_consensus.cc:1261] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 130. Preceding OpId from leader: term: 1 index: 131. (index mismatch)
I20240502 01:58:41.453398 13134 consensus_queue.cc:1035] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 131, Last known committed idx: 130, Time since last communication: 0.000s
I20240502 01:58:41.453465 12590 raft_consensus.cc:1261] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd [term 1 LEARNER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 130. Preceding OpId from leader: term: 1 index: 131. (index mismatch)
I20240502 01:58:41.455801 13134 consensus_queue.cc:1035] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 131, Last known committed idx: 130, Time since last communication: 0.000s
I20240502 01:58:41.456586 13134 raft_consensus.cc:2932] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.131: config changed from index 77 to 131, 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 131 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:41.460307 11418 raft_consensus.cc:2932] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Committing config change with OpId 1.131: config changed from index 77 to 131, 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 131 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:41.460635 11140 catalog_manager.cc:5484] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 reported cstate change: config changed from index 77 to 131, 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 131 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:41.467751 12590 raft_consensus.cc:2932] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Committing config change with OpId 1.131: config changed from index 77 to 131, 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 131 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:41.469275 11285 consensus_queue.cc:237] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 132, Committed index: 132, Last appended: 1.133, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 134 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } }
I20240502 01:58:41.470808 12588 raft_consensus.cc:1261] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 133. Preceding OpId from leader: term: 1 index: 134. (index mismatch)
I20240502 01:58:41.473161 13362 consensus_queue.cc:1035] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 134, Last known committed idx: 132, Time since last communication: 0.000s
W20240502 01:58:41.477532 11246 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
I20240502 01:58:41.479892 13262 raft_consensus.cc:2932] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.134: config changed from index 131 to 134, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New config: { opid_index: 134 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:41.482918 11141 catalog_manager.cc:5484] T 5ddde4eecc19461db50d15139066ea31 P a47375810d964f128aae1e2df1370b79 reported cstate change: config changed from index 131 to 134, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 134 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:41.482986 12588 raft_consensus.cc:2932] T 5ddde4eecc19461db50d15139066ea31 P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Committing config change with OpId 1.134: config changed from index 131 to 134, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New config: { opid_index: 134 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:41.484905 11127 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 5ddde4eecc19461db50d15139066ea31 with cas_config_opid_index 131: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:41.486261 11399 tablet_service.cc:1514] Processing DeleteTablet for tablet 5ddde4eecc19461db50d15139066ea31 with delete_type TABLET_DATA_TOMBSTONED (TS 6bf256be71684cdf87ef861e5890b614 not found in new config with opid_index 134) from {username='slave'} at 127.0.0.1:37868
I20240502 01:58:41.486708 13501 tablet_replica.cc:330] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614: stopping tablet replica
I20240502 01:58:41.486837 13501 raft_consensus.cc:2229] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:41.486922 13501 pending_rounds.cc:62] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614: Trying to abort 1 pending ops.
I20240502 01:58:41.486943 13501 pending_rounds.cc:68] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614: Aborting op as it isn't in flight: id { term: 1 index: 133 } timestamp: 7023063537514045440 op_type: WRITE_OP write_request { tablet_id: "5ddde4eecc19461db50d15139066ea31" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000\252\374\347]\267\212\031Y\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""cd95aeacbbfd8bfe" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7023063537472073728 authz_token { token_data: "\010\371\350\313\261\006\"3\n\005slave\022*\n 3953bb3e36b342bf9945db89810e6cf0\020\001\030\001 \001(\001" signature: "<redacted>""\307n\312\220\253#\327\327e\316\241\030\276\310m\230\370\231\340D\347\273A]\337\360\321\340\327\300\234\275\352\253OK8M7\013\370\'\353\235V-\217`\373K\242}*\360\343\237+bO\356$\013Mm" signing_key_seq_num: 0 } } request_id { client_id: "97ccfc0f71f64ea89955d2414c78dff6" seq_no: 319 first_incomplete_seq_no: 319 attempt_no: 0 }
I20240502 01:58:41.487210 13501 raft_consensus.cc:2258] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:41.488349 13501 ts_tablet_manager.cc:1875] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:41.490336 13501 ts_tablet_manager.cc:1888] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.133
I20240502 01:58:41.490398 13501 log.cc:1196] T 5ddde4eecc19461db50d15139066ea31 P 6bf256be71684cdf87ef861e5890b614: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/5ddde4eecc19461db50d15139066ea31
W20240502 01:58:41.500927 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
I20240502 01:58:41.503291 11128 catalog_manager.cc:4830] TS 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035): tablet 5ddde4eecc19461db50d15139066ea31 (table rebalance_test_table_2 [id=3953bb3e36b342bf9945db89810e6cf0]) successfully deleted
W20240502 01:58:41.524718 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
W20240502 01:58:41.551077 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
W20240502 01:58:41.576591 11247 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Service unavailable: leader transfer in progress
I20240502 01:58:41.598765 13262 raft_consensus.cc:979] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79: : Instructing follower 6bf256be71684cdf87ef861e5890b614 to start an election
I20240502 01:58:41.598904 13262 raft_consensus.cc:1067] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Signalling peer 6bf256be71684cdf87ef861e5890b614 to start an election
I20240502 01:58:41.599438 11420 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "a6b4f69afc6149feac9ca1e82ae995ff"
dest_uuid: "6bf256be71684cdf87ef861e5890b614"
from {username='slave'} at 127.7.221.65:33425
I20240502 01:58:41.599611 11420 raft_consensus.cc:482] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 01:58:41.599671 11420 raft_consensus.cc:3037] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:41.601152 11420 raft_consensus.cc:504] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } }
I20240502 01:58:41.601544 11420 leader_election.cc:288] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [CANDIDATE]: Term 2 election: Requested vote from peers a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857), f271c45722d545f7b33fe7f407444dcb (127.7.221.71:38753)
I20240502 01:58:41.601783 11285 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6b4f69afc6149feac9ca1e82ae995ff" candidate_uuid: "6bf256be71684cdf87ef861e5890b614" candidate_term: 2 candidate_status { last_received { term: 1 index: 94 } } ignore_live_leader: true dest_uuid: "a47375810d964f128aae1e2df1370b79"
I20240502 01:58:41.601919 11285 raft_consensus.cc:3032] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 01:58:41.601984 11285 raft_consensus.cc:729] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: a47375810d964f128aae1e2df1370b79, State: Running, Role: LEADER
I20240502 01:58:41.602147 11285 consensus_queue.cc:260] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 94, Committed index: 94, Last appended: 1.94, Last appended by leader: 94, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } }
I20240502 01:58:41.602309 11285 raft_consensus.cc:3037] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:41.603435 11285 raft_consensus.cc:2454] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6bf256be71684cdf87ef861e5890b614 in term 2.
I20240502 01:58:41.604462 11353 leader_election.cc:302] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6bf256be71684cdf87ef861e5890b614, a47375810d964f128aae1e2df1370b79; no voters:
I20240502 01:58:41.604622 12997 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6b4f69afc6149feac9ca1e82ae995ff" candidate_uuid: "6bf256be71684cdf87ef861e5890b614" candidate_term: 2 candidate_status { last_received { term: 1 index: 94 } } ignore_live_leader: true dest_uuid: "f271c45722d545f7b33fe7f407444dcb"
I20240502 01:58:41.604733 13254 raft_consensus.cc:2781] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 01:58:41.604758 12997 raft_consensus.cc:3037] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:41.604825 13254 raft_consensus.cc:686] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 2 LEADER]: Becoming Leader. State: Replica: 6bf256be71684cdf87ef861e5890b614, State: Running, Role: LEADER
I20240502 01:58:41.604967 13254 consensus_queue.cc:237] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 94, Committed index: 94, Last appended: 1.94, Last appended by leader: 94, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } }
I20240502 01:58:41.606065 12997 raft_consensus.cc:2454] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6bf256be71684cdf87ef861e5890b614 in term 2.
W20240502 01:58:41.607160 11246 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53638: Illegal state: replica a47375810d964f128aae1e2df1370b79 is not leader of this config: current role FOLLOWER
I20240502 01:58:41.607338 11141 catalog_manager.cc:5484] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 reported cstate change: term changed from 1 to 2, leader changed from a47375810d964f128aae1e2df1370b79 (127.7.221.65) to 6bf256be71684cdf87ef861e5890b614 (127.7.221.66). New cstate: current_term: 2 leader_uuid: "6bf256be71684cdf87ef861e5890b614" committed_config { opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:41.638072 11285 raft_consensus.cc:1261] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 2 FOLLOWER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 1 index: 94. Preceding OpId from leader: term: 2 index: 96. (index mismatch)
I20240502 01:58:41.638149 12997 raft_consensus.cc:1261] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 2 FOLLOWER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 1 index: 94. Preceding OpId from leader: term: 2 index: 96. (index mismatch)
I20240502 01:58:41.638623 13124 consensus_queue.cc:1035] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 95, Last known committed idx: 94, Time since last communication: 0.000s
I20240502 01:58:41.638779 13113 consensus_queue.cc:1035] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 95, Last known committed idx: 94, Time since last communication: 0.000s
I20240502 01:58:41.643654 13430 mvcc.cc:205] Tried to move back new op lower bound from 7023063538225487872 to 7023063538095554560. Current Snapshot: MvccSnapshot[applied={T|T < 7023063538225487872 or (T in {7023063538225487872})}]
I20240502 01:58:41.645893 13417 mvcc.cc:205] Tried to move back new op lower bound from 7023063538225487872 to 7023063538095554560. Current Snapshot: MvccSnapshot[applied={T|T < 7023063538225487872}]
I20240502 01:58:41.648730 11418 consensus_queue.cc:237] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 97, Committed index: 97, Last appended: 2.97, Last appended by leader: 94, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 98 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } }
I20240502 01:58:41.650241 12999 raft_consensus.cc:1261] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 2 FOLLOWER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 2 index: 96. Preceding OpId from leader: term: 2 index: 98. (index mismatch)
I20240502 01:58:41.650887 13124 consensus_queue.cc:1035] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 98, Last known committed idx: 96, Time since last communication: 0.000s
I20240502 01:58:41.653815 12997 raft_consensus.cc:1203] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 2 FOLLOWER]: Deduplicated request from leader. Original: 2.96->[2.97-2.98] Dedup: 2.97->[2.98-2.98]
I20240502 01:58:41.655673 13254 raft_consensus.cc:2932] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 2 LEADER]: Committing config change with OpId 2.98: config changed from index 78 to 98, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New config: { opid_index: 98 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:41.657948 11128 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a6b4f69afc6149feac9ca1e82ae995ff with cas_config_opid_index 78: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:41.658143 12997 raft_consensus.cc:2932] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 2 FOLLOWER]: Committing config change with OpId 2.98: config changed from index 78 to 98, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New config: { opid_index: 98 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:41.661538 11140 catalog_manager.cc:5484] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb reported cstate change: config changed from index 78 to 98, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New cstate: current_term: 2 leader_uuid: "6bf256be71684cdf87ef861e5890b614" committed_config { opid_index: 98 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } }
I20240502 01:58:41.664059 13262 raft_consensus.cc:1050] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79: attempting to promote NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 to VOTER
I20240502 01:58:41.664426 13262 consensus_queue.cc:237] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 136, Committed index: 136, Last appended: 1.136, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 137 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } }
I20240502 01:58:41.665513 12723 raft_consensus.cc:1261] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 136. Preceding OpId from leader: term: 1 index: 137. (index mismatch)
I20240502 01:58:41.665874 11548 raft_consensus.cc:1261] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 136. Preceding OpId from leader: term: 1 index: 137. (index mismatch)
I20240502 01:58:41.665894 13263 consensus_queue.cc:1035] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 137, Last known committed idx: 136, Time since last communication: 0.000s
I20240502 01:58:41.666857 13263 consensus_queue.cc:1035] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 137, Last known committed idx: 136, Time since last communication: 0.000s
I20240502 01:58:41.667380 11266 tablet_service.cc:1514] Processing DeleteTablet for tablet a6b4f69afc6149feac9ca1e82ae995ff with delete_type TABLET_DATA_TOMBSTONED (TS a47375810d964f128aae1e2df1370b79 not found in new config with opid_index 98) from {username='slave'} at 127.0.0.1:49072
I20240502 01:58:41.668742 13263 raft_consensus.cc:2932] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.137: config changed from index 104 to 137, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 137 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:41.670789 11548 raft_consensus.cc:2932] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Committing config change with OpId 1.137: config changed from index 104 to 137, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 137 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:41.670892 12723 raft_consensus.cc:2932] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 FOLLOWER]: Committing config change with OpId 1.137: config changed from index 104 to 137, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 137 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:41.673118 11141 catalog_manager.cc:5484] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 reported cstate change: config changed from index 104 to 137, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 137 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:41.678730 13638 tablet_replica.cc:330] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79: stopping tablet replica
I20240502 01:58:41.678916 13638 raft_consensus.cc:2229] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:41.679023 11284 consensus_queue.cc:237] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 138, Committed index: 138, Last appended: 1.138, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } }
I20240502 01:58:41.679085 13638 raft_consensus.cc:2258] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:41.680094 12723 raft_consensus.cc:1261] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 138. Preceding OpId from leader: term: 1 index: 139. (index mismatch)
I20240502 01:58:41.680521 13130 consensus_queue.cc:1035] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 139, Last known committed idx: 138, Time since last communication: 0.000s
I20240502 01:58:41.682762 13262 raft_consensus.cc:2932] T 9375f41e0aad43d3a9d6f5ef7ac73880 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.139: config changed from index 137 to 139, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New config: { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:41.683228 12723 raft_consensus.cc:2932] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 FOLLOWER]: Committing config change with OpId 1.139: config changed from index 137 to 139, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New config: { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:41.684545 11127 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9375f41e0aad43d3a9d6f5ef7ac73880 with cas_config_opid_index 137: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:41.685803 11141 catalog_manager.cc:5484] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 0190cccef0ae48f3baa4123c0e4b49b0 reported cstate change: config changed from index 137 to 139, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:41.689460 11531 tablet_service.cc:1514] Processing DeleteTablet for tablet 9375f41e0aad43d3a9d6f5ef7ac73880 with delete_type TABLET_DATA_TOMBSTONED (TS 8c191817e0ce48b3b552af940f9d11ae not found in new config with opid_index 139) from {username='slave'} at 127.0.0.1:44872
I20240502 01:58:41.698132 13644 tablet_replica.cc:330] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae: stopping tablet replica
I20240502 01:58:41.699484 13644 raft_consensus.cc:2229] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:41.699976 13644 raft_consensus.cc:2258] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:41.700783 13644 ts_tablet_manager.cc:1875] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:41.703609 13644 ts_tablet_manager.cc:1888] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.138
I20240502 01:58:41.707365 13644 log.cc:1196] T 9375f41e0aad43d3a9d6f5ef7ac73880 P 8c191817e0ce48b3b552af940f9d11ae: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/9375f41e0aad43d3a9d6f5ef7ac73880
I20240502 01:58:41.710352 11128 catalog_manager.cc:4830] TS 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991): tablet 9375f41e0aad43d3a9d6f5ef7ac73880 (table rebalance_test_table_1 [id=cb616922efbb491588387c177fbfe8df]) successfully deleted
I20240502 01:58:41.717170 13638 ts_tablet_manager.cc:1875] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:41.720546 13638 ts_tablet_manager.cc:1888] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.97
I20240502 01:58:41.720778 13638 log.cc:1196] T a6b4f69afc6149feac9ca1e82ae995ff P a47375810d964f128aae1e2df1370b79: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/a6b4f69afc6149feac9ca1e82ae995ff
I20240502 01:58:41.721565 11127 catalog_manager.cc:4830] TS a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857): tablet a6b4f69afc6149feac9ca1e82ae995ff (table rebalance_test_table_1 [id=cb616922efbb491588387c177fbfe8df]) successfully deleted
I20240502 01:58:41.959775 12568 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:41.960031 12704 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:41.970513 12840 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:41.974097 11531 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:42.000800 12978 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:42.001408 11399 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:42.009233 11266 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:42.139441 11548 consensus_queue.cc:237] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 136, Committed index: 136, Last appended: 1.137, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 138 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } }
I20240502 01:58:42.140655 12590 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 136. Preceding OpId from leader: term: 1 index: 138. (index mismatch)
I20240502 01:58:42.141052 12287 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 138, Last known committed idx: 136, Time since last communication: 0.000s
W20240502 01:58:42.141973 11485 consensus_peers.cc:485] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae -> Peer 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69:35979): Couldn't send request to peer 0190cccef0ae48f3baa4123c0e4b49b0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e33cb78a2b63498f9c9485b8f751859e. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:42.143748 13135 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.138: config changed from index 92 to 138, NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) added. New config: { opid_index: 138 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } } }
I20240502 01:58:42.144524 12589 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Committing config change with OpId 1.138: config changed from index 92 to 138, NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) added. New config: { opid_index: 138 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } } }
I20240502 01:58:42.146924 11133 catalog_manager.cc:5484] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: config changed from index 92 to 138, NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 138 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:42.201769 11418 consensus_queue.cc:237] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 121, Committed index: 121, Last appended: 2.121, Last appended by leader: 94, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 122 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } }
I20240502 01:58:42.205587 12998 raft_consensus.cc:1261] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 2 FOLLOWER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 2 index: 121. Preceding OpId from leader: term: 2 index: 122. (index mismatch)
I20240502 01:58:42.205971 13113 consensus_queue.cc:1035] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 122, Last known committed idx: 121, Time since last communication: 0.000s
I20240502 01:58:42.209496 13113 raft_consensus.cc:2932] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 2 LEADER]: Committing config change with OpId 2.122: config changed from index 98 to 122, NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) added. New config: { opid_index: 122 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } } }
W20240502 01:58:42.211666 11354 consensus_peers.cc:485] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 -> Peer 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68:43337): Couldn't send request to peer 089e23168d0b48a8baf89ee203e1b4fd. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a6b4f69afc6149feac9ca1e82ae995ff. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:42.213310 11140 catalog_manager.cc:5484] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 reported cstate change: config changed from index 98 to 122, NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) added. New cstate: current_term: 2 leader_uuid: "6bf256be71684cdf87ef861e5890b614" committed_config { opid_index: 122 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:42.214460 12998 raft_consensus.cc:2932] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 2 FOLLOWER]: Committing config change with OpId 2.122: config changed from index 98 to 122, NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) added. New config: { opid_index: 122 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } } }
I20240502 01:58:42.555277 13734 ts_tablet_manager.cc:902] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: Initiating tablet copy from peer 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:42.555799 13734 tablet_copy_client.cc:308] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44991
I20240502 01:58:42.556217 11573 tablet_copy_service.cc:140] P 8c191817e0ce48b3b552af940f9d11ae: Received BeginTabletCopySession request for tablet e33cb78a2b63498f9c9485b8f751859e from peer 0190cccef0ae48f3baa4123c0e4b49b0 ({username='slave'} at 127.7.221.69:45885)
I20240502 01:58:42.556313 11573 tablet_copy_service.cc:161] P 8c191817e0ce48b3b552af940f9d11ae: Beginning new tablet copy session on tablet e33cb78a2b63498f9c9485b8f751859e from peer 0190cccef0ae48f3baa4123c0e4b49b0 at {username='slave'} at 127.7.221.69:45885: session id = 0190cccef0ae48f3baa4123c0e4b49b0-e33cb78a2b63498f9c9485b8f751859e
I20240502 01:58:42.557308 11573 tablet_copy_source_session.cc:205] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:42.561154 13734 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e33cb78a2b63498f9c9485b8f751859e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:42.564023 13734 tablet_copy_client.cc:783] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:42.564200 13734 tablet_copy_client.cc:647] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:42.565824 13734 tablet_copy_client.cc:523] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:42.567260 13734 tablet_bootstrap.cc:492] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: Bootstrap starting.
I20240502 01:58:42.793934 13747 ts_tablet_manager.cc:902] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd: Initiating tablet copy from peer 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035)
I20240502 01:58:42.794492 13747 tablet_copy_client.cc:308] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:36035
I20240502 01:58:42.805171 13734 tablet_bootstrap.cc:492] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: Bootstrap replayed 1/1 log segments. Stats: ops{read=158 overwritten=0 applied=158 ignored=0} inserts{seen=827 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:42.805510 13734 tablet_bootstrap.cc:492] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: Bootstrap complete.
I20240502 01:58:42.805716 13734 ts_tablet_manager.cc:1367] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: Time spent bootstrapping tablet: real 0.238s user 0.120s sys 0.011s
I20240502 01:58:42.806131 13734 raft_consensus.cc:348] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 138 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } }
I20240502 01:58:42.806258 13734 raft_consensus.cc:729] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0190cccef0ae48f3baa4123c0e4b49b0, State: Initialized, Role: LEARNER
I20240502 01:58:42.806378 13734 consensus_queue.cc:260] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 158, Last appended: 1.158, Last appended by leader: 158, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 138 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: true } }
I20240502 01:58:42.806771 13734 ts_tablet_manager.cc:1398] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: Time spent starting tablet: real 0.001s user 0.004s sys 0.000s
I20240502 01:58:42.807154 11573 tablet_copy_service.cc:342] P 8c191817e0ce48b3b552af940f9d11ae: Request end of tablet copy session 0190cccef0ae48f3baa4123c0e4b49b0-e33cb78a2b63498f9c9485b8f751859e received from {username='slave'} at 127.7.221.69:45885
I20240502 01:58:42.807221 11573 tablet_copy_service.cc:433] P 8c191817e0ce48b3b552af940f9d11ae: ending tablet copy session 0190cccef0ae48f3baa4123c0e4b49b0-e33cb78a2b63498f9c9485b8f751859e on tablet e33cb78a2b63498f9c9485b8f751859e with peer 0190cccef0ae48f3baa4123c0e4b49b0
I20240502 01:58:42.823865 11440 tablet_copy_service.cc:140] P 6bf256be71684cdf87ef861e5890b614: Received BeginTabletCopySession request for tablet a6b4f69afc6149feac9ca1e82ae995ff from peer 089e23168d0b48a8baf89ee203e1b4fd ({username='slave'} at 127.7.221.68:56565)
I20240502 01:58:42.824003 11440 tablet_copy_service.cc:161] P 6bf256be71684cdf87ef861e5890b614: Beginning new tablet copy session on tablet a6b4f69afc6149feac9ca1e82ae995ff from peer 089e23168d0b48a8baf89ee203e1b4fd at {username='slave'} at 127.7.221.68:56565: session id = 089e23168d0b48a8baf89ee203e1b4fd-a6b4f69afc6149feac9ca1e82ae995ff
I20240502 01:58:42.824944 11440 tablet_copy_source_session.cc:205] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:42.826655 13747 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6b4f69afc6149feac9ca1e82ae995ff. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:42.828919 13747 tablet_copy_client.cc:783] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:42.829068 13747 tablet_copy_client.cc:647] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:42.843272 13747 tablet_copy_client.cc:523] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:42.844732 13747 tablet_bootstrap.cc:492] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd: Bootstrap starting.
I20240502 01:58:43.020126 13747 tablet_bootstrap.cc:492] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd: Bootstrap replayed 1/1 log segments. Stats: ops{read=134 overwritten=0 applied=134 ignored=0} inserts{seen=815 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:43.020543 13747 tablet_bootstrap.cc:492] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd: Bootstrap complete.
I20240502 01:58:43.020776 13747 ts_tablet_manager.cc:1367] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd: Time spent bootstrapping tablet: real 0.176s user 0.107s sys 0.011s
I20240502 01:58:43.021227 13747 raft_consensus.cc:348] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [term 2 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 122 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } }
I20240502 01:58:43.021366 13747 raft_consensus.cc:729] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [term 2 LEARNER]: Becoming Follower/Learner. State: Replica: 089e23168d0b48a8baf89ee203e1b4fd, State: Initialized, Role: LEARNER
I20240502 01:58:43.021497 13747 consensus_queue.cc:260] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 134, Last appended: 2.134, Last appended by leader: 134, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 122 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: true } }
I20240502 01:58:43.021893 13747 ts_tablet_manager.cc:1398] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:43.025312 11440 tablet_copy_service.cc:342] P 6bf256be71684cdf87ef861e5890b614: Request end of tablet copy session 089e23168d0b48a8baf89ee203e1b4fd-a6b4f69afc6149feac9ca1e82ae995ff received from {username='slave'} at 127.7.221.68:56565
I20240502 01:58:43.025396 11440 tablet_copy_service.cc:433] P 6bf256be71684cdf87ef861e5890b614: ending tablet copy session 089e23168d0b48a8baf89ee203e1b4fd-a6b4f69afc6149feac9ca1e82ae995ff on tablet a6b4f69afc6149feac9ca1e82ae995ff with peer 089e23168d0b48a8baf89ee203e1b4fd
I20240502 01:58:43.139617 12724 raft_consensus.cc:1203] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.137->[1.138-1.185] Dedup: 1.158->[1.159-1.185]
I20240502 01:58:43.158332 13742 raft_consensus.cc:1050] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae: attempting to promote NON_VOTER 0190cccef0ae48f3baa4123c0e4b49b0 to VOTER
I20240502 01:58:43.158689 13742 consensus_queue.cc:237] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 186, Committed index: 186, Last appended: 1.187, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } }
I20240502 01:58:43.159896 12588 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 186. Preceding OpId from leader: term: 1 index: 188. (index mismatch)
I20240502 01:58:43.160655 12383 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 188, Last known committed idx: 186, Time since last communication: 0.000s
I20240502 01:58:43.161525 12722 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 LEARNER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 187. Preceding OpId from leader: term: 1 index: 188. (index mismatch)
I20240502 01:58:43.161922 12588 raft_consensus.cc:1203] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Deduplicated request from leader. Original: 1.186->[1.187-1.188] Dedup: 1.187->[1.188-1.188]
I20240502 01:58:43.162644 12383 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 188, Last known committed idx: 186, Time since last communication: 0.000s
I20240502 01:58:43.162909 13135 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.188: config changed from index 138 to 188, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:43.165221 12588 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Committing config change with OpId 1.188: config changed from index 138 to 188, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:43.165521 11140 catalog_manager.cc:5484] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: config changed from index 138 to 188, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:43.168226 12724 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 FOLLOWER]: Committing config change with OpId 1.188: config changed from index 138 to 188, 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:43.322144 11548 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "e33cb78a2b63498f9c9485b8f751859e"
dest_uuid: "8c191817e0ce48b3b552af940f9d11ae"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:46224
I20240502 01:58:43.322300 11548 raft_consensus.cc:595] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Received request to transfer leadership
W20240502 01:58:43.326185 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.330794 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.337808 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.344470 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.352128 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.361300 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.372404 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.385632 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
I20240502 01:58:43.396832 12589 raft_consensus.cc:1203] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [term 2 LEARNER]: Deduplicated request from leader. Original: 2.121->[2.122-2.157] Dedup: 2.134->[2.135-2.157]
W20240502 01:58:43.397110 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.413414 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.428491 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
I20240502 01:58:43.431660 13669 raft_consensus.cc:1050] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614: attempting to promote NON_VOTER 089e23168d0b48a8baf89ee203e1b4fd to VOTER
I20240502 01:58:43.432067 13669 consensus_queue.cc:237] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 157, Committed index: 157, Last appended: 2.158, Last appended by leader: 94, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 159 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } }
I20240502 01:58:43.433079 12589 raft_consensus.cc:1261] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [term 2 LEARNER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 2 index: 158. Preceding OpId from leader: term: 2 index: 159. (index mismatch)
I20240502 01:58:43.433153 12997 raft_consensus.cc:1261] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 2 FOLLOWER]: Refusing update from remote peer 6bf256be71684cdf87ef861e5890b614: Log matching property violated. Preceding OpId in replica: term: 2 index: 158. Preceding OpId from leader: term: 2 index: 159. (index mismatch)
I20240502 01:58:43.433458 13669 consensus_queue.cc:1035] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 159, Last known committed idx: 157, Time since last communication: 0.000s
I20240502 01:58:43.433585 13113 consensus_queue.cc:1035] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 159, Last known committed idx: 157, Time since last communication: 0.000s
I20240502 01:58:43.435271 13124 raft_consensus.cc:2932] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 2 LEADER]: Committing config change with OpId 2.159: config changed from index 122 to 159, 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 159 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:43.436277 12589 raft_consensus.cc:2932] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [term 2 FOLLOWER]: Committing config change with OpId 2.159: config changed from index 122 to 159, 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 159 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:43.437037 12998 raft_consensus.cc:2932] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 2 FOLLOWER]: Committing config change with OpId 2.159: config changed from index 122 to 159, 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 159 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:43.438911 11133 catalog_manager.cc:5484] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 reported cstate change: config changed from index 122 to 159, 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 2 leader_uuid: "6bf256be71684cdf87ef861e5890b614" committed_config { opid_index: 159 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
W20240502 01:58:43.443329 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.457537 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.475852 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.495455 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.514642 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.531755 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
I20240502 01:58:43.544950 11418 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "a6b4f69afc6149feac9ca1e82ae995ff"
dest_uuid: "6bf256be71684cdf87ef861e5890b614"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:38134
I20240502 01:58:43.545156 11418 raft_consensus.cc:595] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 2 LEADER]: Received request to transfer leadership
W20240502 01:58:43.550681 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.558149 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
I20240502 01:58:43.560977 11548 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "e33cb78a2b63498f9c9485b8f751859e"
dest_uuid: "8c191817e0ce48b3b552af940f9d11ae"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:46248
I20240502 01:58:43.561115 11548 raft_consensus.cc:595] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Received request to transfer leadership
W20240502 01:58:43.563486 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.566545 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.570721 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.573876 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.576016 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.585222 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.592593 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.597381 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.602843 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.614948 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.621548 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.627436 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.641952 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.651186 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.656160 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.669569 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.677495 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.688266 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.702224 11511 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.706313 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.724539 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.728986 11509 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.744019 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.758440 11509 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.765739 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
I20240502 01:58:43.779726 11418 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "a6b4f69afc6149feac9ca1e82ae995ff"
dest_uuid: "6bf256be71684cdf87ef861e5890b614"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:38154
I20240502 01:58:43.779875 11418 raft_consensus.cc:595] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 2 LEADER]: Received request to transfer leadership
W20240502 01:58:43.789356 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.790395 11509 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
I20240502 01:58:43.807250 11548 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "e33cb78a2b63498f9c9485b8f751859e"
dest_uuid: "8c191817e0ce48b3b552af940f9d11ae"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:46266
I20240502 01:58:43.807402 11548 raft_consensus.cc:595] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Received request to transfer leadership
W20240502 01:58:43.813638 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.822261 11509 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
W20240502 01:58:43.835732 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.854180 11509 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Service unavailable: leader transfer in progress
I20240502 01:58:43.854943 13742 raft_consensus.cc:979] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae: : Instructing follower 0190cccef0ae48f3baa4123c0e4b49b0 to start an election
I20240502 01:58:43.855088 13742 raft_consensus.cc:1067] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Signalling peer 0190cccef0ae48f3baa4123c0e4b49b0 to start an election
I20240502 01:58:43.855629 12723 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "e33cb78a2b63498f9c9485b8f751859e"
dest_uuid: "0190cccef0ae48f3baa4123c0e4b49b0"
from {username='slave'} at 127.7.221.67:55005
I20240502 01:58:43.855829 12723 raft_consensus.cc:482] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 01:58:43.855952 12723 raft_consensus.cc:3037] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:43.858110 12723 raft_consensus.cc:504] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } }
W20240502 01:58:43.858763 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
I20240502 01:58:43.859730 12723 leader_election.cc:288] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [CANDIDATE]: Term 2 election: Requested vote from peers 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991), 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68:43337)
I20240502 01:58:43.860235 11548 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e33cb78a2b63498f9c9485b8f751859e" candidate_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" candidate_term: 2 candidate_status { last_received { term: 1 index: 199 } } ignore_live_leader: true dest_uuid: "8c191817e0ce48b3b552af940f9d11ae"
I20240502 01:58:43.860366 11548 raft_consensus.cc:3032] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Stepping down as leader of term 1
I20240502 01:58:43.860407 11548 raft_consensus.cc:729] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Running, Role: LEADER
I20240502 01:58:43.860546 11548 consensus_queue.cc:260] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 199, Committed index: 199, Last appended: 1.199, Last appended by leader: 199, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } }
I20240502 01:58:43.860729 11548 raft_consensus.cc:3037] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:43.861835 11548 raft_consensus.cc:2454] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0190cccef0ae48f3baa4123c0e4b49b0 in term 2.
I20240502 01:58:43.862655 12657 leader_election.cc:302] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0190cccef0ae48f3baa4123c0e4b49b0, 8c191817e0ce48b3b552af940f9d11ae; no voters:
I20240502 01:58:43.863034 13852 raft_consensus.cc:2781] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 01:58:43.863224 13852 raft_consensus.cc:686] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 2 LEADER]: Becoming Leader. State: Replica: 0190cccef0ae48f3baa4123c0e4b49b0, State: Running, Role: LEADER
I20240502 01:58:43.863561 13852 consensus_queue.cc:237] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 199, Committed index: 199, Last appended: 1.199, Last appended by leader: 199, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } }
I20240502 01:58:43.866376 11141 catalog_manager.cc:5484] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 reported cstate change: term changed from 1 to 2, leader changed from 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) to 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69). New cstate: current_term: 2 leader_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" committed_config { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:43.874776 12588 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e33cb78a2b63498f9c9485b8f751859e" candidate_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" candidate_term: 2 candidate_status { last_received { term: 1 index: 199 } } ignore_live_leader: true dest_uuid: "089e23168d0b48a8baf89ee203e1b4fd"
I20240502 01:58:43.874972 12588 raft_consensus.cc:3037] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:43.876364 12588 raft_consensus.cc:2454] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0190cccef0ae48f3baa4123c0e4b49b0 in term 2.
W20240502 01:58:43.883476 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
W20240502 01:58:43.891299 11509 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46012: Illegal state: replica 8c191817e0ce48b3b552af940f9d11ae is not leader of this config: current role FOLLOWER
W20240502 01:58:43.909292 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Service unavailable: leader transfer in progress
I20240502 01:58:43.922191 13124 raft_consensus.cc:979] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614: : Instructing follower 089e23168d0b48a8baf89ee203e1b4fd to start an election
I20240502 01:58:43.922327 13124 raft_consensus.cc:1067] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 2 LEADER]: Signalling peer 089e23168d0b48a8baf89ee203e1b4fd to start an election
I20240502 01:58:43.922973 12590 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "a6b4f69afc6149feac9ca1e82ae995ff"
dest_uuid: "089e23168d0b48a8baf89ee203e1b4fd"
from {username='slave'} at 127.7.221.66:51743
I20240502 01:58:43.923154 12590 raft_consensus.cc:482] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [term 2 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 01:58:43.923219 12590 raft_consensus.cc:3037] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [term 2 FOLLOWER]: Advancing to term 3
I20240502 01:58:43.924304 12590 raft_consensus.cc:504] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [term 3 FOLLOWER]: Starting forced leader election with config: opid_index: 159 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } }
I20240502 01:58:43.928465 11418 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6b4f69afc6149feac9ca1e82ae995ff" candidate_uuid: "089e23168d0b48a8baf89ee203e1b4fd" candidate_term: 3 candidate_status { last_received { term: 2 index: 167 } } ignore_live_leader: true dest_uuid: "6bf256be71684cdf87ef861e5890b614"
I20240502 01:58:43.928772 11418 raft_consensus.cc:3032] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 2 LEADER]: Stepping down as leader of term 2
I20240502 01:58:43.928838 11418 raft_consensus.cc:729] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 2 LEADER]: Becoming Follower/Learner. State: Replica: 6bf256be71684cdf87ef861e5890b614, State: Running, Role: LEADER
I20240502 01:58:43.929052 12590 leader_election.cc:288] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [CANDIDATE]: Term 3 election: Requested vote from peers 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035), f271c45722d545f7b33fe7f407444dcb (127.7.221.71:38753)
I20240502 01:58:43.931488 11418 consensus_queue.cc:260] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 167, Committed index: 167, Last appended: 2.167, Last appended by leader: 167, Current term: 2, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 159 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } }
I20240502 01:58:43.931684 11418 raft_consensus.cc:3037] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 2 FOLLOWER]: Advancing to term 3
I20240502 01:58:43.933046 11418 raft_consensus.cc:2454] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 089e23168d0b48a8baf89ee203e1b4fd in term 3.
I20240502 01:58:43.933719 12521 leader_election.cc:302] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 089e23168d0b48a8baf89ee203e1b4fd, 6bf256be71684cdf87ef861e5890b614; no voters:
I20240502 01:58:43.933992 13765 raft_consensus.cc:2781] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [term 3 FOLLOWER]: Leader election won for term 3
I20240502 01:58:43.934194 13765 raft_consensus.cc:686] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [term 3 LEADER]: Becoming Leader. State: Replica: 089e23168d0b48a8baf89ee203e1b4fd, State: Running, Role: LEADER
I20240502 01:58:43.934500 13765 consensus_queue.cc:237] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 167, Committed index: 167, Last appended: 2.167, Last appended by leader: 167, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 159 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } }
I20240502 01:58:43.935245 12998 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6b4f69afc6149feac9ca1e82ae995ff" candidate_uuid: "089e23168d0b48a8baf89ee203e1b4fd" candidate_term: 3 candidate_status { last_received { term: 2 index: 167 } } ignore_live_leader: true dest_uuid: "f271c45722d545f7b33fe7f407444dcb"
I20240502 01:58:43.935360 12998 raft_consensus.cc:3037] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 2 FOLLOWER]: Advancing to term 3
I20240502 01:58:43.936383 12998 raft_consensus.cc:2454] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 089e23168d0b48a8baf89ee203e1b4fd in term 3.
W20240502 01:58:43.937417 11377 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37974: Illegal state: replica 6bf256be71684cdf87ef861e5890b614 is not leader of this config: current role FOLLOWER
I20240502 01:58:43.937676 11141 catalog_manager.cc:5484] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd reported cstate change: term changed from 2 to 3, leader changed from 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) to 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68). New cstate: current_term: 3 leader_uuid: "089e23168d0b48a8baf89ee203e1b4fd" committed_config { opid_index: 159 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:44.004549 12590 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 2 FOLLOWER]: Refusing update from remote peer 0190cccef0ae48f3baa4123c0e4b49b0: Log matching property violated. Preceding OpId in replica: term: 1 index: 199. Preceding OpId from leader: term: 2 index: 201. (index mismatch)
I20240502 01:58:44.004732 11548 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 2 FOLLOWER]: Refusing update from remote peer 0190cccef0ae48f3baa4123c0e4b49b0: Log matching property violated. Preceding OpId in replica: term: 1 index: 199. Preceding OpId from leader: term: 2 index: 201. (index mismatch)
I20240502 01:58:44.007783 13852 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 200, Last known committed idx: 199, Time since last communication: 0.000s
I20240502 01:58:44.008201 13852 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 200, Last known committed idx: 199, Time since last communication: 0.000s
I20240502 01:58:44.015061 12997 raft_consensus.cc:1261] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 3 FOLLOWER]: Refusing update from remote peer 089e23168d0b48a8baf89ee203e1b4fd: Log matching property violated. Preceding OpId in replica: term: 2 index: 167. Preceding OpId from leader: term: 3 index: 169. (index mismatch)
I20240502 01:58:44.015130 11418 raft_consensus.cc:1261] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 3 FOLLOWER]: Refusing update from remote peer 089e23168d0b48a8baf89ee203e1b4fd: Log matching property violated. Preceding OpId in replica: term: 2 index: 167. Preceding OpId from leader: term: 3 index: 169. (index mismatch)
I20240502 01:58:44.015645 13765 consensus_queue.cc:1035] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [LEADER]: Connected to new peer: Peer: permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 168, Last known committed idx: 167, Time since last communication: 0.000s
I20240502 01:58:44.015978 13857 consensus_queue.cc:1035] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 168, Last known committed idx: 167, Time since last communication: 0.000s
I20240502 01:58:44.019872 13859 mvcc.cc:205] Tried to move back new op lower bound from 7023063547960680448 to 7023063547638935552. Current Snapshot: MvccSnapshot[applied={T|T < 7023063547960680448 or (T in {7023063547960680448})}]
I20240502 01:58:44.047422 12588 consensus_queue.cc:237] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 171, Committed index: 171, Last appended: 3.171, Last appended by leader: 167, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 172 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } }
I20240502 01:58:44.051116 12998 raft_consensus.cc:1261] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 3 FOLLOWER]: Refusing update from remote peer 089e23168d0b48a8baf89ee203e1b4fd: Log matching property violated. Preceding OpId in replica: term: 3 index: 171. Preceding OpId from leader: term: 3 index: 172. (index mismatch)
I20240502 01:58:44.053020 13869 consensus_queue.cc:1035] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [LEADER]: Connected to new peer: Peer: permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 172, Last known committed idx: 171, Time since last communication: 0.000s
I20240502 01:58:44.055078 13765 raft_consensus.cc:2932] T a6b4f69afc6149feac9ca1e82ae995ff P 089e23168d0b48a8baf89ee203e1b4fd [term 3 LEADER]: Committing config change with OpId 3.172: config changed from index 159 to 172, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New config: { opid_index: 172 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:44.056610 11128 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a6b4f69afc6149feac9ca1e82ae995ff with cas_config_opid_index 159: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:44.057600 12998 raft_consensus.cc:2932] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb [term 3 FOLLOWER]: Committing config change with OpId 3.172: config changed from index 159 to 172, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New config: { opid_index: 172 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:44.060549 11133 catalog_manager.cc:5484] T a6b4f69afc6149feac9ca1e82ae995ff P f271c45722d545f7b33fe7f407444dcb reported cstate change: config changed from index 159 to 172, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New cstate: current_term: 3 leader_uuid: "089e23168d0b48a8baf89ee203e1b4fd" committed_config { opid_index: 172 OBSOLETE_local: false peers { permanent_uuid: "f271c45722d545f7b33fe7f407444dcb" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 38753 } attrs { promote: false } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } }
I20240502 01:58:44.061007 12722 consensus_queue.cc:237] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 202, Committed index: 202, Last appended: 2.202, Last appended by leader: 199, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 203 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } }
I20240502 01:58:44.062965 12588 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 2 FOLLOWER]: Refusing update from remote peer 0190cccef0ae48f3baa4123c0e4b49b0: Log matching property violated. Preceding OpId in replica: term: 2 index: 202. Preceding OpId from leader: term: 2 index: 203. (index mismatch)
I20240502 01:58:44.065289 11399 tablet_service.cc:1514] Processing DeleteTablet for tablet a6b4f69afc6149feac9ca1e82ae995ff with delete_type TABLET_DATA_TOMBSTONED (TS 6bf256be71684cdf87ef861e5890b614 not found in new config with opid_index 172) from {username='slave'} at 127.0.0.1:37868
I20240502 01:58:44.066141 13853 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 203, Last known committed idx: 202, Time since last communication: 0.000s
I20240502 01:58:44.069814 13874 tablet_replica.cc:330] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614: stopping tablet replica
I20240502 01:58:44.069989 13874 raft_consensus.cc:2229] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 3 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:44.070391 13874 pending_rounds.cc:62] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614: Trying to abort 1 pending ops.
I20240502 01:58:44.070454 13874 pending_rounds.cc:68] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614: Aborting op as it isn't in flight: id { term: 3 index: 171 } timestamp: 7023063548069543936 op_type: WRITE_OP write_request { tablet_id: "a6b4f69afc6149feac9ca1e82ae995ff" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000\314ge\036\216{\202\237\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""7356c4640aef3722" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7023063548041121792 authz_token { token_data: "\010\371\350\313\261\006\"3\n\005slave\022*\n cb616922efbb491588387c177fbfe8df\020\001\030\001 \001(\001" signature: "<redacted>""\274I\270\347\\K\263\342\254\323\272]:\350\335C\355\366w \265DV\t\223\213x\221\263Bu\004\350\316\3732\324/p\177.\357\036\037,\214\r\024\3417\303\\lli\371\245j\010\021\204\033\335\352" signing_key_seq_num: 0 } } request_id { client_id: "4607b83acc8442fb90e2341da3bc4495" seq_no: 565 first_incomplete_seq_no: 565 attempt_no: 0 }
I20240502 01:58:44.070966 13874 raft_consensus.cc:2258] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614 [term 3 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:44.075378 13874 ts_tablet_manager.cc:1875] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:44.078078 13874 ts_tablet_manager.cc:1888] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 3.171
I20240502 01:58:44.078222 13874 log.cc:1196] T a6b4f69afc6149feac9ca1e82ae995ff P 6bf256be71684cdf87ef861e5890b614: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/a6b4f69afc6149feac9ca1e82ae995ff
I20240502 01:58:44.080480 11128 catalog_manager.cc:4830] TS 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035): tablet a6b4f69afc6149feac9ca1e82ae995ff (table rebalance_test_table_1 [id=cb616922efbb491588387c177fbfe8df]) successfully deleted
I20240502 01:58:44.082520 13853 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 2 LEADER]: Committing config change with OpId 2.203: config changed from index 188 to 203, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New config: { opid_index: 203 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:44.084192 11127 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e33cb78a2b63498f9c9485b8f751859e with cas_config_opid_index 188: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:44.085559 11141 catalog_manager.cc:5484] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 reported cstate change: config changed from index 188 to 203, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New cstate: current_term: 2 leader_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" committed_config { opid_index: 203 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:44.085611 12589 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 2 FOLLOWER]: Committing config change with OpId 2.203: config changed from index 188 to 203, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New config: { opid_index: 203 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false } } }
I20240502 01:58:44.098949 11531 tablet_service.cc:1514] Processing DeleteTablet for tablet e33cb78a2b63498f9c9485b8f751859e with delete_type TABLET_DATA_TOMBSTONED (TS 8c191817e0ce48b3b552af940f9d11ae not found in new config with opid_index 203) from {username='slave'} at 127.0.0.1:44872
I20240502 01:58:44.099450 13875 tablet_replica.cc:330] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae: stopping tablet replica
I20240502 01:58:44.099597 13875 raft_consensus.cc:2229] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:44.099922 13875 raft_consensus.cc:2258] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:44.101222 13875 ts_tablet_manager.cc:1875] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:44.114463 13875 ts_tablet_manager.cc:1888] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.202
I20240502 01:58:44.114545 13875 log.cc:1196] T e33cb78a2b63498f9c9485b8f751859e P 8c191817e0ce48b3b552af940f9d11ae: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/e33cb78a2b63498f9c9485b8f751859e
I20240502 01:58:44.116039 11128 catalog_manager.cc:4830] TS 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991): tablet e33cb78a2b63498f9c9485b8f751859e (table rebalance_test_table_3 [id=d37cf611838446c7a7e593d1673fa1df]) successfully deleted
I20240502 01:58:44.358803 12704 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:44.362344 12568 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:44.413120 11399 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 01:58:44.415912 11266 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:44.426362 11531 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:44.426998 12840 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:44.428012 12978 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:44.728554 12704 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:44.742600 12840 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:44.744984 11266 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:44.749544 11399 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 01:58:44.763209 12568 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:44.777731 12978 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:44.779417 11531 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:45.281414 12704 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:45.286803 12568 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:45.294817 11399 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 01:58:45.348555 11531 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:45.348855 11266 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:45.350720 12840 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:45.359287 12978 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:45.646500 12704 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:45.647326 12840 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:45.687430 11266 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:45.689679 12568 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:45.712795 11531 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:45.727213 12978 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:45.749665 11399 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 01:58:45.935653 12704 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:45.939234 12840 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:45.940260 11399 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 01:58:45.944595 12978 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:45.950058 11531 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:45.958103 12568 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:45.963805 11266 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
629fef51a73345408bea08d1a5ecc913 | 127.7.221.126:32951 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
allow_unsafe_replication_factor | true | runtime,unsafe | all 1 server(s) checked
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.7.221.84:39105 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+---------+----------+----------------+-----------------
0190cccef0ae48f3baa4123c0e4b49b0 | 127.7.221.69:35979 | HEALTHY | <none> | 1 | 0
089e23168d0b48a8baf89ee203e1b4fd | 127.7.221.68:43337 | HEALTHY | <none> | 1 | 0
6bf256be71684cdf87ef861e5890b614 | 127.7.221.66:36035 | HEALTHY | <none> | 3 | 0
76bcefd6842e4c09ba16b8b6aa1ef662 | 127.7.221.70:46849 | HEALTHY | <none> | 1 | 0
8c191817e0ce48b3b552af940f9d11ae | 127.7.221.67:44991 | HEALTHY | <none> | 4 | 0
a47375810d964f128aae1e2df1370b79 | 127.7.221.65:36857 | HEALTHY | <none> | 5 | 0
f271c45722d545f7b33fe7f407444dcb | 127.7.221.71:38753 | HEALTHY | <none> | 1 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 7
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 7 server(s) checked
local_ip_for_outbound_sockets | 127.7.221.65 | experimental | 127.7.221.65:36857
local_ip_for_outbound_sockets | 127.7.221.66 | experimental | 127.7.221.66:36035
local_ip_for_outbound_sockets | 127.7.221.67 | experimental | 127.7.221.67:44991
local_ip_for_outbound_sockets | 127.7.221.68 | experimental | 127.7.221.68:43337
local_ip_for_outbound_sockets | 127.7.221.69 | experimental | 127.7.221.69:35979
local_ip_for_outbound_sockets | 127.7.221.70 | experimental | 127.7.221.70:46849
local_ip_for_outbound_sockets | 127.7.221.71 | experimental | 127.7.221.71:38753
never_fsync | true | unsafe,advanced | all 7 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 7 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 7 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 7 server(s) checked
server_dump_info_format | pb | hidden | all 7 server(s) checked
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.7.221.65:36857
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.7.221.66:36035
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.7.221.67:44991
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.7.221.68:43337
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.7.221.69:35979
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.7.221.70:46849
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/info.pb | hidden | 127.7.221.71:38753
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.7.221.84:39105 | all 7 server(s) checked
time_source | builtin | all 7 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 8 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
TestTable | 2 | HEALTHY | 1 | 1 | 0 | 0 | 0
rebalance_test_table_0 | 2 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 2 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 2 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 2 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_4 | 2 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 4
First Quartile | 4
Median | 5
Third Quartile | 5
Maximum | 5
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 7
Tables | 6
Tablets | 16
Replicas | 32
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240502 01:58:45.988337 8053 log_verifier.cc:126] Checking tablet 055b7173c3174a019457ad258989238a
I20240502 01:58:46.040727 8053 log_verifier.cc:177] Verified matching terms for 276 ops in tablet 055b7173c3174a019457ad258989238a
I20240502 01:58:46.040951 8053 log_verifier.cc:126] Checking tablet 0f1917ed678a4b8798e917d9c4e511da
I20240502 01:58:46.091068 8053 log_verifier.cc:177] Verified matching terms for 272 ops in tablet 0f1917ed678a4b8798e917d9c4e511da
I20240502 01:58:46.091204 8053 log_verifier.cc:126] Checking tablet 1568709dd2424f188f484adba0c720d4
I20240502 01:58:46.144349 8053 log_verifier.cc:177] Verified matching terms for 287 ops in tablet 1568709dd2424f188f484adba0c720d4
I20240502 01:58:46.144491 8053 log_verifier.cc:126] Checking tablet 1819f2f9c847491886d78211e3ca0fcc
I20240502 01:58:46.158267 8053 log_verifier.cc:177] Verified matching terms for 2 ops in tablet 1819f2f9c847491886d78211e3ca0fcc
I20240502 01:58:46.158341 8053 log_verifier.cc:126] Checking tablet 2dd97f65baa14cd9b3010b5e20592f1f
I20240502 01:58:46.207350 8053 log_verifier.cc:177] Verified matching terms for 234 ops in tablet 2dd97f65baa14cd9b3010b5e20592f1f
I20240502 01:58:46.207469 8053 log_verifier.cc:126] Checking tablet 495cddfb76b241a89676a723a9cc8b44
I20240502 01:58:46.259235 8053 log_verifier.cc:177] Verified matching terms for 277 ops in tablet 495cddfb76b241a89676a723a9cc8b44
I20240502 01:58:46.259373 8053 log_verifier.cc:126] Checking tablet 4ba576fba5b44e149c1c6a302248fd0f
I20240502 01:58:46.312574 8053 log_verifier.cc:177] Verified matching terms for 248 ops in tablet 4ba576fba5b44e149c1c6a302248fd0f
I20240502 01:58:46.312713 8053 log_verifier.cc:126] Checking tablet 4cef15d5c5fc41cbb55d184731eff9e2
I20240502 01:58:46.364892 8053 log_verifier.cc:177] Verified matching terms for 261 ops in tablet 4cef15d5c5fc41cbb55d184731eff9e2
I20240502 01:58:46.365007 8053 log_verifier.cc:126] Checking tablet 5ddde4eecc19461db50d15139066ea31
I20240502 01:58:46.417083 8053 log_verifier.cc:177] Verified matching terms for 280 ops in tablet 5ddde4eecc19461db50d15139066ea31
I20240502 01:58:46.417198 8053 log_verifier.cc:126] Checking tablet 706c5ef6b1b24ce28a70cef754b74d73
I20240502 01:58:46.460067 8053 log_verifier.cc:177] Verified matching terms for 213 ops in tablet 706c5ef6b1b24ce28a70cef754b74d73
I20240502 01:58:46.460165 8053 log_verifier.cc:126] Checking tablet 91e43d66f1664c7db8f1f16f623640be
I20240502 01:58:46.515808 8053 log_verifier.cc:177] Verified matching terms for 256 ops in tablet 91e43d66f1664c7db8f1f16f623640be
I20240502 01:58:46.515928 8053 log_verifier.cc:126] Checking tablet 9375f41e0aad43d3a9d6f5ef7ac73880
I20240502 01:58:46.567849 8053 log_verifier.cc:177] Verified matching terms for 287 ops in tablet 9375f41e0aad43d3a9d6f5ef7ac73880
I20240502 01:58:46.567962 8053 log_verifier.cc:126] Checking tablet a1596ccad20249358ffbd0a4e3a5e1cb
I20240502 01:58:46.613623 8053 log_verifier.cc:177] Verified matching terms for 252 ops in tablet a1596ccad20249358ffbd0a4e3a5e1cb
I20240502 01:58:46.613744 8053 log_verifier.cc:126] Checking tablet a408622a1ff546a6b70bb23101521b90
I20240502 01:58:46.671066 8053 log_verifier.cc:177] Verified matching terms for 283 ops in tablet a408622a1ff546a6b70bb23101521b90
I20240502 01:58:46.671216 8053 log_verifier.cc:126] Checking tablet a6b4f69afc6149feac9ca1e82ae995ff
I20240502 01:58:46.718855 8053 log_verifier.cc:177] Verified matching terms for 251 ops in tablet a6b4f69afc6149feac9ca1e82ae995ff
I20240502 01:58:46.718988 8053 log_verifier.cc:126] Checking tablet e33cb78a2b63498f9c9485b8f751859e
I20240502 01:58:46.767400 8053 log_verifier.cc:177] Verified matching terms for 257 ops in tablet e33cb78a2b63498f9c9485b8f751859e
I20240502 01:58:46.769388 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.72:0
--local_ip_for_outbound_sockets=127.7.221.72
--webserver_interface=127.7.221.72
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--builtin_ntp_servers=127.7.221.84:39105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 01:58:46.872838 14095 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:46.873039 14095 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:46.873077 14095 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:46.876009 14095 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:46.876063 14095 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:46.876139 14095 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.72
I20240502 01:58:46.879699 14095 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.72:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.7.221.72
--webserver_port=0
--tserver_master_addrs=127.7.221.126:32951
--never_fsync=true
--heap_profile_path=/tmp/kudu.14095
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.72
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:46.880576 14095 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:46.881717 14095 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:46.890264 14104 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:46.891167 14095 server_base.cc:1034] running on GCE node
W20240502 01:58:46.891744 14102 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:46.892576 14101 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:46.892967 14095 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:46.893492 14095 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:46.894662 14095 hybrid_clock.cc:648] HybridClock initialized: now 1714615126894644 us; error 32 us; skew 500 ppm
I20240502 01:58:46.896653 14095 webserver.cc:460] Webserver started at http://127.7.221.72:45193/ using document root <none> and password file <none>
I20240502 01:58:46.897217 14095 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:46.897275 14095 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:46.897466 14095 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:46.899030 14095 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data/instance:
uuid: "c9900a57125149e69854e49405383e15"
format_stamp: "Formatted at 2024-05-02 01:58:46 on dist-test-slave-dmxt"
I20240502 01:58:46.899492 14095 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal/instance:
uuid: "c9900a57125149e69854e49405383e15"
format_stamp: "Formatted at 2024-05-02 01:58:46 on dist-test-slave-dmxt"
I20240502 01:58:46.902482 14095 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.001s
I20240502 01:58:46.904671 14111 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:46.905642 14095 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:46.905741 14095 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "c9900a57125149e69854e49405383e15"
format_stamp: "Formatted at 2024-05-02 01:58:46 on dist-test-slave-dmxt"
I20240502 01:58:46.905823 14095 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:46.918725 14095 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:46.919349 14095 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:46.919494 14095 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:46.919960 14095 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:46.920831 14095 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:46.920876 14095 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:46.920933 14095 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:46.920957 14095 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:46.928939 14095 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.72:38577
I20240502 01:58:46.928967 14226 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.72:38577 every 8 connection(s)
I20240502 01:58:46.929854 14095 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 01:58:46.935494 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 14095
I20240502 01:58:46.935632 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal/instance
I20240502 01:58:46.938789 14227 heartbeater.cc:347] Connected to a master server at 127.7.221.126:32951
I20240502 01:58:46.939008 14227 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:46.939505 14227 heartbeater.cc:511] Master 127.7.221.126:32951 requested a full tablet report, sending...
I20240502 01:58:46.940507 11133 ts_manager.cc:194] Registered new tserver with Master: c9900a57125149e69854e49405383e15 (127.7.221.72:38577)
I20240502 01:58:46.941092 11133 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.72:43065
I20240502 01:58:47.119721 12704 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:47.120023 12568 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:47.122205 11399 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 01:58:47.130339 11266 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:47.136070 11531 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:47.138689 14159 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:47.147353 12840 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:47.149475 12978 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:47.236261 11399 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 01:58:47.240792 12704 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:47.241130 11531 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:47.244267 12568 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:47.245852 12840 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:47.257087 11266 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 01:58:47.262833 14159 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:47.275178 12978 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:47.304584 11284 consensus_queue.cc:237] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 287, Committed index: 287, Last appended: 1.287, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 288 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } }
I20240502 01:58:47.306139 12861 raft_consensus.cc:1261] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 287. Preceding OpId from leader: term: 1 index: 288. (index mismatch)
I20240502 01:58:47.306558 14019 consensus_queue.cc:1035] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 288, Last known committed idx: 287, Time since last communication: 0.000s
I20240502 01:58:47.308287 14328 raft_consensus.cc:2932] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.288: config changed from index 95 to 288, NON_VOTER c9900a57125149e69854e49405383e15 (127.7.221.72) added. New config: { opid_index: 288 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } } }
I20240502 01:58:47.308815 12861 raft_consensus.cc:2932] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Committing config change with OpId 1.288: config changed from index 95 to 288, NON_VOTER c9900a57125149e69854e49405383e15 (127.7.221.72) added. New config: { opid_index: 288 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } } }
W20240502 01:58:47.311300 11220 consensus_peers.cc:485] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 -> Peer c9900a57125149e69854e49405383e15 (127.7.221.72:38577): Couldn't send request to peer c9900a57125149e69854e49405383e15. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1568709dd2424f188f484adba0c720d4. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:47.311427 11142 catalog_manager.cc:5484] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 reported cstate change: config changed from index 95 to 288, NON_VOTER c9900a57125149e69854e49405383e15 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 288 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } } }
I20240502 01:58:47.320319 11552 consensus_queue.cc:237] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 252, Committed index: 252, Last appended: 1.252, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 253 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } }
I20240502 01:58:47.321691 12589 raft_consensus.cc:1261] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 252. Preceding OpId from leader: term: 1 index: 253. (index mismatch)
I20240502 01:58:47.322137 14057 consensus_queue.cc:1035] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 253, Last known committed idx: 252, Time since last communication: 0.000s
I20240502 01:58:47.323704 14341 raft_consensus.cc:2932] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.253: config changed from index 70 to 253, NON_VOTER c9900a57125149e69854e49405383e15 (127.7.221.72) added. New config: { opid_index: 253 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } } }
I20240502 01:58:47.324122 12589 raft_consensus.cc:2932] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Committing config change with OpId 1.253: config changed from index 70 to 253, NON_VOTER c9900a57125149e69854e49405383e15 (127.7.221.72) added. New config: { opid_index: 253 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } } }
W20240502 01:58:47.326258 11485 consensus_peers.cc:485] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae -> Peer c9900a57125149e69854e49405383e15 (127.7.221.72:38577): Couldn't send request to peer c9900a57125149e69854e49405383e15. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a1596ccad20249358ffbd0a4e3a5e1cb. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:47.326869 11143 catalog_manager.cc:5484] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd reported cstate change: config changed from index 70 to 253, NON_VOTER c9900a57125149e69854e49405383e15 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 253 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } } }
I20240502 01:58:47.333734 12861 consensus_queue.cc:237] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 234, Committed index: 234, Last appended: 2.234, Last appended by leader: 85, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 235 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } }
I20240502 01:58:47.335155 11419 raft_consensus.cc:1261] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 2 FOLLOWER]: Refusing update from remote peer 76bcefd6842e4c09ba16b8b6aa1ef662: Log matching property violated. Preceding OpId in replica: term: 2 index: 234. Preceding OpId from leader: term: 2 index: 235. (index mismatch)
I20240502 01:58:47.335620 14350 consensus_queue.cc:1035] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 235, Last known committed idx: 234, Time since last communication: 0.000s
I20240502 01:58:47.337453 14233 raft_consensus.cc:2932] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 2 LEADER]: Committing config change with OpId 2.235: config changed from index 89 to 235, NON_VOTER c9900a57125149e69854e49405383e15 (127.7.221.72) added. New config: { opid_index: 235 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } } }
I20240502 01:58:47.337904 11419 raft_consensus.cc:2932] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 2 FOLLOWER]: Committing config change with OpId 2.235: config changed from index 89 to 235, NON_VOTER c9900a57125149e69854e49405383e15 (127.7.221.72) added. New config: { opid_index: 235 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } } }
W20240502 01:58:47.338814 12792 consensus_peers.cc:485] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 -> Peer c9900a57125149e69854e49405383e15 (127.7.221.72:38577): Couldn't send request to peer c9900a57125149e69854e49405383e15. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2dd97f65baa14cd9b3010b5e20592f1f. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:47.339970 11133 catalog_manager.cc:5484] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 reported cstate change: config changed from index 89 to 235, NON_VOTER c9900a57125149e69854e49405383e15 (127.7.221.72) added. New cstate: current_term: 2 leader_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" committed_config { opid_index: 235 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:47.347219 12724 consensus_queue.cc:237] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 257, Committed index: 257, Last appended: 2.257, Last appended by leader: 199, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 258 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } }
I20240502 01:58:47.348567 12589 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 2 FOLLOWER]: Refusing update from remote peer 0190cccef0ae48f3baa4123c0e4b49b0: Log matching property violated. Preceding OpId in replica: term: 2 index: 257. Preceding OpId from leader: term: 2 index: 258. (index mismatch)
I20240502 01:58:47.348968 14359 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 258, Last known committed idx: 257, Time since last communication: 0.000s
I20240502 01:58:47.350628 14310 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 2 LEADER]: Committing config change with OpId 2.258: config changed from index 203 to 258, NON_VOTER c9900a57125149e69854e49405383e15 (127.7.221.72) added. New config: { opid_index: 258 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } } }
I20240502 01:58:47.351040 12589 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 2 FOLLOWER]: Committing config change with OpId 2.258: config changed from index 203 to 258, NON_VOTER c9900a57125149e69854e49405383e15 (127.7.221.72) added. New config: { opid_index: 258 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } } }
W20240502 01:58:47.351531 12656 consensus_peers.cc:485] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 -> Peer c9900a57125149e69854e49405383e15 (127.7.221.72:38577): Couldn't send request to peer c9900a57125149e69854e49405383e15. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e33cb78a2b63498f9c9485b8f751859e. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:58:47.353132 11143 catalog_manager.cc:5484] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 reported cstate change: config changed from index 203 to 258, NON_VOTER c9900a57125149e69854e49405383e15 (127.7.221.72) added. New cstate: current_term: 2 leader_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" committed_config { opid_index: 258 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:47.771826 14403 ts_tablet_manager.cc:902] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15: Initiating tablet copy from peer a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857)
I20240502 01:58:47.772483 14403 tablet_copy_client.cc:308] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:36857
I20240502 01:58:47.775926 11307 tablet_copy_service.cc:140] P a47375810d964f128aae1e2df1370b79: Received BeginTabletCopySession request for tablet 1568709dd2424f188f484adba0c720d4 from peer c9900a57125149e69854e49405383e15 ({username='slave'} at 127.7.221.72:43357)
I20240502 01:58:47.776015 11307 tablet_copy_service.cc:161] P a47375810d964f128aae1e2df1370b79: Beginning new tablet copy session on tablet 1568709dd2424f188f484adba0c720d4 from peer c9900a57125149e69854e49405383e15 at {username='slave'} at 127.7.221.72:43357: session id = c9900a57125149e69854e49405383e15-1568709dd2424f188f484adba0c720d4
I20240502 01:58:47.776700 11307 tablet_copy_source_session.cc:205] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:47.779453 14403 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1568709dd2424f188f484adba0c720d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:47.784112 14403 tablet_copy_client.cc:783] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:47.784278 14403 tablet_copy_client.cc:647] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:47.786317 14403 tablet_copy_client.cc:523] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:47.787811 14403 tablet_bootstrap.cc:492] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15: Bootstrap starting.
I20240502 01:58:47.813233 14403 log.cc:824] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:47.882002 14432 ts_tablet_manager.cc:902] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15: Initiating tablet copy from peer 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70:46849)
I20240502 01:58:47.882558 14432 tablet_copy_client.cc:308] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.70:46849
I20240502 01:58:47.886040 12881 tablet_copy_service.cc:140] P 76bcefd6842e4c09ba16b8b6aa1ef662: Received BeginTabletCopySession request for tablet 2dd97f65baa14cd9b3010b5e20592f1f from peer c9900a57125149e69854e49405383e15 ({username='slave'} at 127.7.221.72:56713)
I20240502 01:58:47.886150 12881 tablet_copy_service.cc:161] P 76bcefd6842e4c09ba16b8b6aa1ef662: Beginning new tablet copy session on tablet 2dd97f65baa14cd9b3010b5e20592f1f from peer c9900a57125149e69854e49405383e15 at {username='slave'} at 127.7.221.72:56713: session id = c9900a57125149e69854e49405383e15-2dd97f65baa14cd9b3010b5e20592f1f
I20240502 01:58:47.887262 12881 tablet_copy_source_session.cc:205] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:47.888361 14432 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2dd97f65baa14cd9b3010b5e20592f1f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:47.890389 14432 tablet_copy_client.cc:783] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:47.890678 14432 tablet_copy_client.cc:647] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:47.892369 14432 tablet_copy_client.cc:523] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:47.893389 14432 tablet_bootstrap.cc:492] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15: Bootstrap starting.
I20240502 01:58:47.910023 14439 ts_tablet_manager.cc:902] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15: Initiating tablet copy from peer 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69:35979)
I20240502 01:58:47.910414 14440 ts_tablet_manager.cc:902] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15: Initiating tablet copy from peer 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991)
I20240502 01:58:47.910555 14439 tablet_copy_client.cc:308] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.69:35979
I20240502 01:58:47.910949 14440 tablet_copy_client.cc:308] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44991
I20240502 01:58:47.915825 11573 tablet_copy_service.cc:140] P 8c191817e0ce48b3b552af940f9d11ae: Received BeginTabletCopySession request for tablet a1596ccad20249358ffbd0a4e3a5e1cb from peer c9900a57125149e69854e49405383e15 ({username='slave'} at 127.7.221.72:34517)
I20240502 01:58:47.915922 11573 tablet_copy_service.cc:161] P 8c191817e0ce48b3b552af940f9d11ae: Beginning new tablet copy session on tablet a1596ccad20249358ffbd0a4e3a5e1cb from peer c9900a57125149e69854e49405383e15 at {username='slave'} at 127.7.221.72:34517: session id = c9900a57125149e69854e49405383e15-a1596ccad20249358ffbd0a4e3a5e1cb
I20240502 01:58:47.915938 12745 tablet_copy_service.cc:140] P 0190cccef0ae48f3baa4123c0e4b49b0: Received BeginTabletCopySession request for tablet e33cb78a2b63498f9c9485b8f751859e from peer c9900a57125149e69854e49405383e15 ({username='slave'} at 127.7.221.72:38579)
I20240502 01:58:47.916067 12745 tablet_copy_service.cc:161] P 0190cccef0ae48f3baa4123c0e4b49b0: Beginning new tablet copy session on tablet e33cb78a2b63498f9c9485b8f751859e from peer c9900a57125149e69854e49405383e15 at {username='slave'} at 127.7.221.72:38579: session id = c9900a57125149e69854e49405383e15-e33cb78a2b63498f9c9485b8f751859e
I20240502 01:58:47.916754 11573 tablet_copy_source_session.cc:205] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:47.917384 12745 tablet_copy_source_session.cc:205] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:58:47.917521 14440 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a1596ccad20249358ffbd0a4e3a5e1cb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:47.918475 14439 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e33cb78a2b63498f9c9485b8f751859e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:47.919519 14440 tablet_copy_client.cc:783] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:47.919662 14440 tablet_copy_client.cc:647] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:47.920528 14439 tablet_copy_client.cc:783] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15: tablet copy: Starting download of 0 data blocks...
I20240502 01:58:47.920712 14439 tablet_copy_client.cc:647] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15: tablet copy: Starting download of 1 WAL segments...
I20240502 01:58:47.921716 14440 tablet_copy_client.cc:523] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:47.922806 14440 tablet_bootstrap.cc:492] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15: Bootstrap starting.
I20240502 01:58:47.922945 14439 tablet_copy_client.cc:523] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:58:47.923976 14439 tablet_bootstrap.cc:492] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15: Bootstrap starting.
I20240502 01:58:47.942768 14227 heartbeater.cc:503] Master 127.7.221.126:32951 was elected leader, sending a full tablet report...
I20240502 01:58:47.993839 14403 tablet_bootstrap.cc:492] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15: Bootstrap replayed 1/1 log segments. Stats: ops{read=288 overwritten=0 applied=288 ignored=0} inserts{seen=970 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:47.994328 14403 tablet_bootstrap.cc:492] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15: Bootstrap complete.
I20240502 01:58:47.994866 14403 ts_tablet_manager.cc:1367] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15: Time spent bootstrapping tablet: real 0.207s user 0.185s sys 0.004s
I20240502 01:58:47.998317 14403 raft_consensus.cc:348] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 288 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } }
I20240502 01:58:47.998672 14403 raft_consensus.cc:729] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: c9900a57125149e69854e49405383e15, State: Initialized, Role: LEARNER
I20240502 01:58:47.999068 14403 consensus_queue.cc:260] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 288, Last appended: 1.288, Last appended by leader: 288, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 288 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } }
I20240502 01:58:48.000059 14403 ts_tablet_manager.cc:1398] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15: Time spent starting tablet: real 0.005s user 0.004s sys 0.000s
I20240502 01:58:48.002050 11307 tablet_copy_service.cc:342] P a47375810d964f128aae1e2df1370b79: Request end of tablet copy session c9900a57125149e69854e49405383e15-1568709dd2424f188f484adba0c720d4 received from {username='slave'} at 127.7.221.72:43357
I20240502 01:58:48.002146 11307 tablet_copy_service.cc:433] P a47375810d964f128aae1e2df1370b79: ending tablet copy session c9900a57125149e69854e49405383e15-1568709dd2424f188f484adba0c720d4 on tablet 1568709dd2424f188f484adba0c720d4 with peer c9900a57125149e69854e49405383e15
I20240502 01:58:48.075556 14432 tablet_bootstrap.cc:492] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15: Bootstrap replayed 1/1 log segments. Stats: ops{read=235 overwritten=0 applied=235 ignored=0} inserts{seen=903 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:48.075984 14432 tablet_bootstrap.cc:492] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15: Bootstrap complete.
I20240502 01:58:48.076251 14432 ts_tablet_manager.cc:1367] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15: Time spent bootstrapping tablet: real 0.183s user 0.160s sys 0.012s
I20240502 01:58:48.077328 14432 raft_consensus.cc:348] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15 [term 2 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 235 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } }
I20240502 01:58:48.077529 14432 raft_consensus.cc:729] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15 [term 2 LEARNER]: Becoming Follower/Learner. State: Replica: c9900a57125149e69854e49405383e15, State: Initialized, Role: LEARNER
I20240502 01:58:48.077709 14432 consensus_queue.cc:260] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 235, Last appended: 2.235, Last appended by leader: 235, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 235 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } }
I20240502 01:58:48.078064 14432 ts_tablet_manager.cc:1398] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:48.078696 12881 tablet_copy_service.cc:342] P 76bcefd6842e4c09ba16b8b6aa1ef662: Request end of tablet copy session c9900a57125149e69854e49405383e15-2dd97f65baa14cd9b3010b5e20592f1f received from {username='slave'} at 127.7.221.72:56713
I20240502 01:58:48.078801 12881 tablet_copy_service.cc:433] P 76bcefd6842e4c09ba16b8b6aa1ef662: ending tablet copy session c9900a57125149e69854e49405383e15-2dd97f65baa14cd9b3010b5e20592f1f on tablet 2dd97f65baa14cd9b3010b5e20592f1f with peer c9900a57125149e69854e49405383e15
I20240502 01:58:48.113003 14439 tablet_bootstrap.cc:492] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15: Bootstrap replayed 1/1 log segments. Stats: ops{read=258 overwritten=0 applied=258 ignored=0} inserts{seen=923 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:48.113307 14440 tablet_bootstrap.cc:492] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15: Bootstrap replayed 1/1 log segments. Stats: ops{read=253 overwritten=0 applied=253 ignored=0} inserts{seen=922 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:58:48.113368 14439 tablet_bootstrap.cc:492] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15: Bootstrap complete.
I20240502 01:58:48.113548 14439 ts_tablet_manager.cc:1367] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15: Time spent bootstrapping tablet: real 0.190s user 0.156s sys 0.023s
I20240502 01:58:48.113606 14440 tablet_bootstrap.cc:492] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15: Bootstrap complete.
I20240502 01:58:48.113818 14440 ts_tablet_manager.cc:1367] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15: Time spent bootstrapping tablet: real 0.191s user 0.163s sys 0.012s
I20240502 01:58:48.113999 14439 raft_consensus.cc:348] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [term 2 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 258 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } }
I20240502 01:58:48.114168 14439 raft_consensus.cc:729] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [term 2 LEARNER]: Becoming Follower/Learner. State: Replica: c9900a57125149e69854e49405383e15, State: Initialized, Role: LEARNER
I20240502 01:58:48.114192 14440 raft_consensus.cc:348] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 253 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } }
I20240502 01:58:48.114305 14440 raft_consensus.cc:729] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: c9900a57125149e69854e49405383e15, State: Initialized, Role: LEARNER
I20240502 01:58:48.114276 14439 consensus_queue.cc:260] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 258, Last appended: 2.258, Last appended by leader: 258, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 258 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } }
I20240502 01:58:48.114415 14440 consensus_queue.cc:260] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 253, Last appended: 1.253, Last appended by leader: 253, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 253 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: true } }
I20240502 01:58:48.114564 14439 ts_tablet_manager.cc:1398] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:48.114718 14440 ts_tablet_manager.cc:1398] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:48.115223 11573 tablet_copy_service.cc:342] P 8c191817e0ce48b3b552af940f9d11ae: Request end of tablet copy session c9900a57125149e69854e49405383e15-a1596ccad20249358ffbd0a4e3a5e1cb received from {username='slave'} at 127.7.221.72:34517
I20240502 01:58:48.115315 11573 tablet_copy_service.cc:433] P 8c191817e0ce48b3b552af940f9d11ae: ending tablet copy session c9900a57125149e69854e49405383e15-a1596ccad20249358ffbd0a4e3a5e1cb on tablet a1596ccad20249358ffbd0a4e3a5e1cb with peer c9900a57125149e69854e49405383e15
I20240502 01:58:48.115309 12745 tablet_copy_service.cc:342] P 0190cccef0ae48f3baa4123c0e4b49b0: Request end of tablet copy session c9900a57125149e69854e49405383e15-e33cb78a2b63498f9c9485b8f751859e received from {username='slave'} at 127.7.221.72:38579
I20240502 01:58:48.115417 12745 tablet_copy_service.cc:433] P 0190cccef0ae48f3baa4123c0e4b49b0: ending tablet copy session c9900a57125149e69854e49405383e15-e33cb78a2b63498f9c9485b8f751859e on tablet e33cb78a2b63498f9c9485b8f751859e with peer c9900a57125149e69854e49405383e15
I20240502 01:58:48.188894 14179 raft_consensus.cc:1203] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.287->[1.288-1.288] Dedup: 1.288->[]
I20240502 01:58:48.385169 14179 raft_consensus.cc:1203] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15 [term 2 LEARNER]: Deduplicated request from leader. Original: 2.234->[2.235-2.235] Dedup: 2.235->[]
I20240502 01:58:48.440444 14179 raft_consensus.cc:1203] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [term 2 LEARNER]: Deduplicated request from leader. Original: 2.257->[2.258-2.258] Dedup: 2.258->[]
I20240502 01:58:48.492033 14179 raft_consensus.cc:1203] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.252->[1.253-1.253] Dedup: 1.253->[]
I20240502 01:58:48.729799 14545 raft_consensus.cc:1050] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79: attempting to promote NON_VOTER c9900a57125149e69854e49405383e15 to VOTER
I20240502 01:58:48.730175 14545 consensus_queue.cc:237] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 288, Committed index: 288, Last appended: 1.288, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 289 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:48.731385 14179 raft_consensus.cc:1261] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [term 1 LEARNER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 288. Preceding OpId from leader: term: 1 index: 289. (index mismatch)
I20240502 01:58:48.731416 12861 raft_consensus.cc:1261] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Refusing update from remote peer a47375810d964f128aae1e2df1370b79: Log matching property violated. Preceding OpId in replica: term: 1 index: 288. Preceding OpId from leader: term: 1 index: 289. (index mismatch)
I20240502 01:58:48.731760 14547 consensus_queue.cc:1035] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 289, Last known committed idx: 288, Time since last communication: 0.000s
I20240502 01:58:48.731940 14545 consensus_queue.cc:1035] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 289, Last known committed idx: 288, Time since last communication: 0.000s
I20240502 01:58:48.733547 14547 raft_consensus.cc:2932] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Committing config change with OpId 1.289: config changed from index 288 to 289, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 289 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:48.734019 12861 raft_consensus.cc:2932] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Committing config change with OpId 1.289: config changed from index 288 to 289, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 289 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:48.736685 11133 catalog_manager.cc:5484] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 reported cstate change: config changed from index 288 to 289, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "a47375810d964f128aae1e2df1370b79" committed_config { opid_index: 289 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:48.739279 14179 raft_consensus.cc:2932] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [term 1 FOLLOWER]: Committing config change with OpId 1.289: config changed from index 288 to 289, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 289 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:48.812562 14496 raft_consensus.cc:1050] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662: attempting to promote NON_VOTER c9900a57125149e69854e49405383e15 to VOTER
I20240502 01:58:48.812963 14496 consensus_queue.cc:237] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 235, Committed index: 235, Last appended: 2.235, Last appended by leader: 85, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 236 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:48.814239 11419 raft_consensus.cc:1261] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 2 FOLLOWER]: Refusing update from remote peer 76bcefd6842e4c09ba16b8b6aa1ef662: Log matching property violated. Preceding OpId in replica: term: 2 index: 235. Preceding OpId from leader: term: 2 index: 236. (index mismatch)
I20240502 01:58:48.814347 14179 raft_consensus.cc:1261] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15 [term 2 LEARNER]: Refusing update from remote peer 76bcefd6842e4c09ba16b8b6aa1ef662: Log matching property violated. Preceding OpId in replica: term: 2 index: 235. Preceding OpId from leader: term: 2 index: 236. (index mismatch)
I20240502 01:58:48.814700 14497 consensus_queue.cc:1035] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 236, Last known committed idx: 235, Time since last communication: 0.000s
I20240502 01:58:48.815208 14497 consensus_queue.cc:1035] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 236, Last known committed idx: 235, Time since last communication: 0.000s
I20240502 01:58:48.816637 14498 raft_consensus.cc:2932] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 2 LEADER]: Committing config change with OpId 2.236: config changed from index 235 to 236, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 236 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:48.817078 11419 raft_consensus.cc:2932] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 2 FOLLOWER]: Committing config change with OpId 2.236: config changed from index 235 to 236, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 236 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:48.817448 14179 raft_consensus.cc:2932] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15 [term 2 FOLLOWER]: Committing config change with OpId 2.236: config changed from index 235 to 236, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 236 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:48.819490 11142 catalog_manager.cc:5484] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15 reported cstate change: config changed from index 235 to 236, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New cstate: current_term: 2 leader_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" committed_config { opid_index: 236 OBSOLETE_local: false peers { permanent_uuid: "6bf256be71684cdf87ef861e5890b614" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 36035 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:48.822405 12861 consensus_queue.cc:237] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 236, Committed index: 236, Last appended: 2.236, Last appended by leader: 85, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 237 OBSOLETE_local: false peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:48.823233 14179 raft_consensus.cc:1261] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15 [term 2 FOLLOWER]: Refusing update from remote peer 76bcefd6842e4c09ba16b8b6aa1ef662: Log matching property violated. Preceding OpId in replica: term: 2 index: 236. Preceding OpId from leader: term: 2 index: 237. (index mismatch)
I20240502 01:58:48.823542 14497 consensus_queue.cc:1035] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 237, Last known committed idx: 236, Time since last communication: 0.000s
I20240502 01:58:48.824838 14496 raft_consensus.cc:2932] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 2 LEADER]: Committing config change with OpId 2.237: config changed from index 236 to 237, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New config: { opid_index: 237 OBSOLETE_local: false peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:48.825253 14179 raft_consensus.cc:2932] T 2dd97f65baa14cd9b3010b5e20592f1f P c9900a57125149e69854e49405383e15 [term 2 FOLLOWER]: Committing config change with OpId 2.237: config changed from index 236 to 237, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New config: { opid_index: 237 OBSOLETE_local: false peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:48.826371 11129 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2dd97f65baa14cd9b3010b5e20592f1f with cas_config_opid_index 236: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:48.826828 11133 catalog_manager.cc:5484] T 2dd97f65baa14cd9b3010b5e20592f1f P 76bcefd6842e4c09ba16b8b6aa1ef662 reported cstate change: config changed from index 236 to 237, VOTER 6bf256be71684cdf87ef861e5890b614 (127.7.221.66) evicted. New cstate: current_term: 2 leader_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" committed_config { opid_index: 237 OBSOLETE_local: false peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:48.828922 11399 tablet_service.cc:1514] Processing DeleteTablet for tablet 2dd97f65baa14cd9b3010b5e20592f1f with delete_type TABLET_DATA_TOMBSTONED (TS 6bf256be71684cdf87ef861e5890b614 not found in new config with opid_index 237) from {username='slave'} at 127.0.0.1:37868
I20240502 01:58:48.829224 14558 tablet_replica.cc:330] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614: stopping tablet replica
I20240502 01:58:48.829358 14558 raft_consensus.cc:2229] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:48.829522 14558 raft_consensus.cc:2258] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:48.831008 14558 ts_tablet_manager.cc:1875] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:48.832640 14558 ts_tablet_manager.cc:1888] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.236
I20240502 01:58:48.832702 14558 log.cc:1196] T 2dd97f65baa14cd9b3010b5e20592f1f P 6bf256be71684cdf87ef861e5890b614: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/2dd97f65baa14cd9b3010b5e20592f1f
I20240502 01:58:48.833158 11128 catalog_manager.cc:4830] TS 6bf256be71684cdf87ef861e5890b614 (127.7.221.66:36035): tablet 2dd97f65baa14cd9b3010b5e20592f1f (table rebalance_test_table_2 [id=3953bb3e36b342bf9945db89810e6cf0]) successfully deleted
I20240502 01:58:48.946911 11284 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "1568709dd2424f188f484adba0c720d4"
dest_uuid: "a47375810d964f128aae1e2df1370b79"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:34534
I20240502 01:58:48.947038 11284 raft_consensus.cc:595] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:58:48.954269 14589 raft_consensus.cc:1050] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: attempting to promote NON_VOTER c9900a57125149e69854e49405383e15 to VOTER
I20240502 01:58:48.954937 14589 consensus_queue.cc:237] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 258, Committed index: 258, Last appended: 2.258, Last appended by leader: 199, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 259 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:48.956236 14179 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [term 2 LEARNER]: Refusing update from remote peer 0190cccef0ae48f3baa4123c0e4b49b0: Log matching property violated. Preceding OpId in replica: term: 2 index: 258. Preceding OpId from leader: term: 2 index: 259. (index mismatch)
I20240502 01:58:48.956365 12589 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 2 FOLLOWER]: Refusing update from remote peer 0190cccef0ae48f3baa4123c0e4b49b0: Log matching property violated. Preceding OpId in replica: term: 2 index: 258. Preceding OpId from leader: term: 2 index: 259. (index mismatch)
I20240502 01:58:48.956732 14589 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 259, Last known committed idx: 258, Time since last communication: 0.000s
I20240502 01:58:48.956985 14500 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 259, Last known committed idx: 258, Time since last communication: 0.000s
I20240502 01:58:48.958787 14500 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 2 LEADER]: Committing config change with OpId 2.259: config changed from index 258 to 259, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 259 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:48.959194 14179 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [term 2 FOLLOWER]: Committing config change with OpId 2.259: config changed from index 258 to 259, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 259 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:48.960991 12589 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 2 FOLLOWER]: Committing config change with OpId 2.259: config changed from index 258 to 259, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 259 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:48.961210 11143 catalog_manager.cc:5484] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 reported cstate change: config changed from index 258 to 259, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New cstate: current_term: 2 leader_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" committed_config { opid_index: 259 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:48.971671 11284 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "1568709dd2424f188f484adba0c720d4"
dest_uuid: "a47375810d964f128aae1e2df1370b79"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:34560
I20240502 01:58:48.971786 11284 raft_consensus.cc:595] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:58:48.997071 14519 raft_consensus.cc:1050] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae: attempting to promote NON_VOTER c9900a57125149e69854e49405383e15 to VOTER
I20240502 01:58:48.997416 14519 consensus_queue.cc:237] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 253, Committed index: 253, Last appended: 1.253, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 254 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:48.998519 14179 raft_consensus.cc:1261] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15 [term 1 LEARNER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 253. Preceding OpId from leader: term: 1 index: 254. (index mismatch)
I20240502 01:58:48.998615 12589 raft_consensus.cc:1261] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Refusing update from remote peer 8c191817e0ce48b3b552af940f9d11ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 253. Preceding OpId from leader: term: 1 index: 254. (index mismatch)
I20240502 01:58:48.998939 14519 consensus_queue.cc:1035] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 254, Last known committed idx: 253, Time since last communication: 0.000s
I20240502 01:58:48.999212 14611 consensus_queue.cc:1035] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 254, Last known committed idx: 253, Time since last communication: 0.000s
I20240502 01:58:49.000962 14613 raft_consensus.cc:2932] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Committing config change with OpId 1.254: config changed from index 253 to 254, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 254 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:49.001389 12589 raft_consensus.cc:2932] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Committing config change with OpId 1.254: config changed from index 253 to 254, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 254 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:49.002969 14179 raft_consensus.cc:2932] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15 [term 1 FOLLOWER]: Committing config change with OpId 1.254: config changed from index 253 to 254, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 254 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:49.003619 11139 catalog_manager.cc:5484] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae reported cstate change: config changed from index 253 to 254, c9900a57125149e69854e49405383e15 (127.7.221.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8c191817e0ce48b3b552af940f9d11ae" committed_config { opid_index: 254 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:49.186933 12724 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "e33cb78a2b63498f9c9485b8f751859e"
dest_uuid: "0190cccef0ae48f3baa4123c0e4b49b0"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:58358
I20240502 01:58:49.187130 12724 raft_consensus.cc:595] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 2 LEADER]: Received request to transfer leadership
I20240502 01:58:49.196352 11284 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "1568709dd2424f188f484adba0c720d4"
dest_uuid: "a47375810d964f128aae1e2df1370b79"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:34590
I20240502 01:58:49.196483 11284 raft_consensus.cc:595] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:58:49.205473 11552 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "a1596ccad20249358ffbd0a4e3a5e1cb"
dest_uuid: "8c191817e0ce48b3b552af940f9d11ae"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:42520
I20240502 01:58:49.205592 11552 raft_consensus.cc:595] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Received request to transfer leadership
I20240502 01:58:49.282792 14475 raft_consensus.cc:979] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79: : Instructing follower c9900a57125149e69854e49405383e15 to start an election
I20240502 01:58:49.282891 14475 raft_consensus.cc:1067] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Signalling peer c9900a57125149e69854e49405383e15 to start an election
I20240502 01:58:49.283362 14179 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "1568709dd2424f188f484adba0c720d4"
dest_uuid: "c9900a57125149e69854e49405383e15"
from {username='slave'} at 127.7.221.65:34383
I20240502 01:58:49.283543 14179 raft_consensus.cc:482] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 01:58:49.283661 14179 raft_consensus.cc:3037] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:49.284575 14179 raft_consensus.cc:504] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 289 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:49.285593 14179 leader_election.cc:288] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [CANDIDATE]: Term 2 election: Requested vote from peers a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857), 76bcefd6842e4c09ba16b8b6aa1ef662 (127.7.221.70:46849)
I20240502 01:58:49.286098 12861 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1568709dd2424f188f484adba0c720d4" candidate_uuid: "c9900a57125149e69854e49405383e15" candidate_term: 2 candidate_status { last_received { term: 1 index: 289 } } ignore_live_leader: true dest_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662"
I20240502 01:58:49.286120 11284 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1568709dd2424f188f484adba0c720d4" candidate_uuid: "c9900a57125149e69854e49405383e15" candidate_term: 2 candidate_status { last_received { term: 1 index: 289 } } ignore_live_leader: true dest_uuid: "a47375810d964f128aae1e2df1370b79"
I20240502 01:58:49.286223 12861 raft_consensus.cc:3037] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:49.286231 11284 raft_consensus.cc:3032] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 01:58:49.286266 11284 raft_consensus.cc:729] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: a47375810d964f128aae1e2df1370b79, State: Running, Role: LEADER
I20240502 01:58:49.286403 11284 consensus_queue.cc:260] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 289, Committed index: 289, Last appended: 1.289, Last appended by leader: 289, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 289 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:49.286566 11284 raft_consensus.cc:3037] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:49.287144 12861 raft_consensus.cc:2454] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9900a57125149e69854e49405383e15 in term 2.
I20240502 01:58:49.287513 11284 raft_consensus.cc:2454] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9900a57125149e69854e49405383e15 in term 2.
I20240502 01:58:49.287513 14115 leader_election.cc:302] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 76bcefd6842e4c09ba16b8b6aa1ef662, c9900a57125149e69854e49405383e15; no voters:
I20240502 01:58:49.287712 14453 raft_consensus.cc:2781] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 01:58:49.287918 14453 raft_consensus.cc:686] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [term 2 LEADER]: Becoming Leader. State: Replica: c9900a57125149e69854e49405383e15, State: Running, Role: LEADER
I20240502 01:58:49.288213 14453 consensus_queue.cc:237] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 289, Committed index: 289, Last appended: 1.289, Last appended by leader: 289, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 289 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:49.290187 11133 catalog_manager.cc:5484] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 reported cstate change: term changed from 1 to 2, leader changed from a47375810d964f128aae1e2df1370b79 (127.7.221.65) to c9900a57125149e69854e49405383e15 (127.7.221.72). New cstate: current_term: 2 leader_uuid: "c9900a57125149e69854e49405383e15" committed_config { opid_index: 289 OBSOLETE_local: false peers { permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:49.397359 14592 raft_consensus.cc:979] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: : Instructing follower c9900a57125149e69854e49405383e15 to start an election
I20240502 01:58:49.397498 14592 raft_consensus.cc:1067] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 2 LEADER]: Signalling peer c9900a57125149e69854e49405383e15 to start an election
I20240502 01:58:49.398047 14179 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "e33cb78a2b63498f9c9485b8f751859e"
dest_uuid: "c9900a57125149e69854e49405383e15"
from {username='slave'} at 127.7.221.69:41575
I20240502 01:58:49.398177 14179 raft_consensus.cc:482] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [term 2 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 01:58:49.398241 14179 raft_consensus.cc:3037] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [term 2 FOLLOWER]: Advancing to term 3
I20240502 01:58:49.399097 14179 raft_consensus.cc:504] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [term 3 FOLLOWER]: Starting forced leader election with config: opid_index: 259 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:49.399574 14179 leader_election.cc:288] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [CANDIDATE]: Term 3 election: Requested vote from peers 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68:43337), 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69:35979)
I20240502 01:58:49.400040 12724 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e33cb78a2b63498f9c9485b8f751859e" candidate_uuid: "c9900a57125149e69854e49405383e15" candidate_term: 3 candidate_status { last_received { term: 2 index: 259 } } ignore_live_leader: true dest_uuid: "0190cccef0ae48f3baa4123c0e4b49b0"
I20240502 01:58:49.400223 12724 raft_consensus.cc:3032] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 2 LEADER]: Stepping down as leader of term 2
I20240502 01:58:49.400281 12724 raft_consensus.cc:729] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 2 LEADER]: Becoming Follower/Learner. State: Replica: 0190cccef0ae48f3baa4123c0e4b49b0, State: Running, Role: LEADER
I20240502 01:58:49.400444 12724 consensus_queue.cc:260] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 259, Committed index: 259, Last appended: 2.259, Last appended by leader: 259, Current term: 2, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 259 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:49.400637 12724 raft_consensus.cc:3037] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 2 FOLLOWER]: Advancing to term 3
I20240502 01:58:49.401397 12724 raft_consensus.cc:2454] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9900a57125149e69854e49405383e15 in term 3.
I20240502 01:58:49.401751 14113 leader_election.cc:302] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0190cccef0ae48f3baa4123c0e4b49b0, c9900a57125149e69854e49405383e15; no voters:
I20240502 01:58:49.401926 14453 raft_consensus.cc:2781] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [term 3 FOLLOWER]: Leader election won for term 3
I20240502 01:58:49.401998 14453 raft_consensus.cc:686] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [term 3 LEADER]: Becoming Leader. State: Replica: c9900a57125149e69854e49405383e15, State: Running, Role: LEADER
I20240502 01:58:49.402160 14453 consensus_queue.cc:237] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 259, Committed index: 259, Last appended: 2.259, Last appended by leader: 259, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 259 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:49.403592 12589 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e33cb78a2b63498f9c9485b8f751859e" candidate_uuid: "c9900a57125149e69854e49405383e15" candidate_term: 3 candidate_status { last_received { term: 2 index: 259 } } ignore_live_leader: true dest_uuid: "089e23168d0b48a8baf89ee203e1b4fd"
I20240502 01:58:49.403719 12589 raft_consensus.cc:3037] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 2 FOLLOWER]: Advancing to term 3
I20240502 01:58:49.403867 11133 catalog_manager.cc:5484] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 reported cstate change: term changed from 2 to 3, leader changed from 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) to c9900a57125149e69854e49405383e15 (127.7.221.72). New cstate: current_term: 3 leader_uuid: "c9900a57125149e69854e49405383e15" committed_config { opid_index: 259 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:49.404508 12589 raft_consensus.cc:2454] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9900a57125149e69854e49405383e15 in term 3.
I20240502 01:58:49.428668 11552 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "a1596ccad20249358ffbd0a4e3a5e1cb"
dest_uuid: "8c191817e0ce48b3b552af940f9d11ae"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:42534
I20240502 01:58:49.428787 11552 raft_consensus.cc:595] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Received request to transfer leadership
I20240502 01:58:49.439126 14611 raft_consensus.cc:979] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae: : Instructing follower 089e23168d0b48a8baf89ee203e1b4fd to start an election
I20240502 01:58:49.439235 14611 raft_consensus.cc:1067] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Signalling peer 089e23168d0b48a8baf89ee203e1b4fd to start an election
I20240502 01:58:49.439590 12589 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "a1596ccad20249358ffbd0a4e3a5e1cb"
dest_uuid: "089e23168d0b48a8baf89ee203e1b4fd"
from {username='slave'} at 127.7.221.67:51109
I20240502 01:58:49.439703 12589 raft_consensus.cc:482] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 01:58:49.439762 12589 raft_consensus.cc:3037] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:49.440513 12589 raft_consensus.cc:504] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 254 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:49.440925 12589 leader_election.cc:288] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [CANDIDATE]: Term 2 election: Requested vote from peers 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991), c9900a57125149e69854e49405383e15 (127.7.221.72:38577)
I20240502 01:58:49.441241 11552 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a1596ccad20249358ffbd0a4e3a5e1cb" candidate_uuid: "089e23168d0b48a8baf89ee203e1b4fd" candidate_term: 2 candidate_status { last_received { term: 1 index: 254 } } ignore_live_leader: true dest_uuid: "8c191817e0ce48b3b552af940f9d11ae"
I20240502 01:58:49.441357 11552 raft_consensus.cc:3032] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Stepping down as leader of term 1
I20240502 01:58:49.441399 11552 raft_consensus.cc:729] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 8c191817e0ce48b3b552af940f9d11ae, State: Running, Role: LEADER
I20240502 01:58:49.441558 11552 consensus_queue.cc:260] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 254, Committed index: 254, Last appended: 1.254, Last appended by leader: 254, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 254 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:49.441704 11552 raft_consensus.cc:3037] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:49.442672 11552 raft_consensus.cc:2454] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 089e23168d0b48a8baf89ee203e1b4fd in term 2.
I20240502 01:58:49.442967 12521 leader_election.cc:302] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 089e23168d0b48a8baf89ee203e1b4fd, 8c191817e0ce48b3b552af940f9d11ae; no voters:
I20240502 01:58:49.443153 14595 raft_consensus.cc:2781] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 2 FOLLOWER]: Leader election won for term 2
I20240502 01:58:49.443332 14595 raft_consensus.cc:686] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 2 LEADER]: Becoming Leader. State: Replica: 089e23168d0b48a8baf89ee203e1b4fd, State: Running, Role: LEADER
I20240502 01:58:49.443481 14595 consensus_queue.cc:237] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 254, Committed index: 254, Last appended: 1.254, Last appended by leader: 254, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 254 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:49.444757 14179 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a1596ccad20249358ffbd0a4e3a5e1cb" candidate_uuid: "089e23168d0b48a8baf89ee203e1b4fd" candidate_term: 2 candidate_status { last_received { term: 1 index: 254 } } ignore_live_leader: true dest_uuid: "c9900a57125149e69854e49405383e15"
I20240502 01:58:49.444905 14179 raft_consensus.cc:3037] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:58:49.445585 14179 raft_consensus.cc:2454] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 089e23168d0b48a8baf89ee203e1b4fd in term 2.
I20240502 01:58:49.446230 11133 catalog_manager.cc:5484] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd reported cstate change: term changed from 1 to 2, leader changed from 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) to 089e23168d0b48a8baf89ee203e1b4fd (127.7.221.68). New cstate: current_term: 2 leader_uuid: "089e23168d0b48a8baf89ee203e1b4fd" committed_config { opid_index: 254 OBSOLETE_local: false peers { permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:49.801776 12589 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 3 FOLLOWER]: Refusing update from remote peer c9900a57125149e69854e49405383e15: Log matching property violated. Preceding OpId in replica: term: 2 index: 259. Preceding OpId from leader: term: 3 index: 260. (index mismatch)
I20240502 01:58:49.802312 14453 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 260, Last known committed idx: 259, Time since last communication: 0.000s
I20240502 01:58:49.805409 11284 raft_consensus.cc:1261] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 2 FOLLOWER]: Refusing update from remote peer c9900a57125149e69854e49405383e15: Log matching property violated. Preceding OpId in replica: term: 1 index: 289. Preceding OpId from leader: term: 2 index: 290. (index mismatch)
I20240502 01:58:49.805490 12724 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 3 FOLLOWER]: Refusing update from remote peer c9900a57125149e69854e49405383e15: Log matching property violated. Preceding OpId in replica: term: 2 index: 259. Preceding OpId from leader: term: 3 index: 260. (index mismatch)
I20240502 01:58:49.806020 14453 consensus_queue.cc:1035] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a47375810d964f128aae1e2df1370b79" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36857 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 290, Last known committed idx: 289, Time since last communication: 0.000s
I20240502 01:58:49.806313 14453 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0190cccef0ae48f3baa4123c0e4b49b0" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35979 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 260, Last known committed idx: 259, Time since last communication: 0.000s
I20240502 01:58:49.809885 12861 raft_consensus.cc:1261] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 2 FOLLOWER]: Refusing update from remote peer c9900a57125149e69854e49405383e15: Log matching property violated. Preceding OpId in replica: term: 1 index: 289. Preceding OpId from leader: term: 2 index: 290. (index mismatch)
I20240502 01:58:49.810257 14699 consensus_queue.cc:1035] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 290, Last known committed idx: 289, Time since last communication: 0.000s
I20240502 01:58:49.812863 14179 consensus_queue.cc:237] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 260, Committed index: 260, Last appended: 3.260, Last appended by leader: 259, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 261 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:49.813946 12589 raft_consensus.cc:1261] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 3 FOLLOWER]: Refusing update from remote peer c9900a57125149e69854e49405383e15: Log matching property violated. Preceding OpId in replica: term: 3 index: 260. Preceding OpId from leader: term: 3 index: 261. (index mismatch)
I20240502 01:58:49.814299 14453 consensus_queue.cc:1035] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 261, Last known committed idx: 260, Time since last communication: 0.000s
I20240502 01:58:49.815726 14699 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 [term 3 LEADER]: Committing config change with OpId 3.261: config changed from index 259 to 261, VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) evicted. New config: { opid_index: 261 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:49.816136 12589 raft_consensus.cc:2932] T e33cb78a2b63498f9c9485b8f751859e P 089e23168d0b48a8baf89ee203e1b4fd [term 3 FOLLOWER]: Committing config change with OpId 3.261: config changed from index 259 to 261, VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) evicted. New config: { opid_index: 261 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:49.816992 11127 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e33cb78a2b63498f9c9485b8f751859e with cas_config_opid_index 259: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:49.817792 11133 catalog_manager.cc:5484] T e33cb78a2b63498f9c9485b8f751859e P c9900a57125149e69854e49405383e15 reported cstate change: config changed from index 259 to 261, VOTER 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69) evicted. New cstate: current_term: 3 leader_uuid: "c9900a57125149e69854e49405383e15" committed_config { opid_index: 261 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:49.820093 12704 tablet_service.cc:1514] Processing DeleteTablet for tablet e33cb78a2b63498f9c9485b8f751859e with delete_type TABLET_DATA_TOMBSTONED (TS 0190cccef0ae48f3baa4123c0e4b49b0 not found in new config with opid_index 261) from {username='slave'} at 127.0.0.1:40844
I20240502 01:58:49.820529 14703 tablet_replica.cc:330] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: stopping tablet replica
I20240502 01:58:49.820847 14703 raft_consensus.cc:2229] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 3 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:49.821004 14703 raft_consensus.cc:2258] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0 [term 3 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:49.822499 14703 ts_tablet_manager.cc:1875] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:49.824256 14703 ts_tablet_manager.cc:1888] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 3.260
I20240502 01:58:49.824347 14703 log.cc:1196] T e33cb78a2b63498f9c9485b8f751859e P 0190cccef0ae48f3baa4123c0e4b49b0: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/wals/e33cb78a2b63498f9c9485b8f751859e
I20240502 01:58:49.824904 11127 catalog_manager.cc:4830] TS 0190cccef0ae48f3baa4123c0e4b49b0 (127.7.221.69:35979): tablet e33cb78a2b63498f9c9485b8f751859e (table rebalance_test_table_3 [id=d37cf611838446c7a7e593d1673fa1df]) successfully deleted
I20240502 01:58:49.871927 11552 raft_consensus.cc:1261] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 2 FOLLOWER]: Refusing update from remote peer 089e23168d0b48a8baf89ee203e1b4fd: Log matching property violated. Preceding OpId in replica: term: 1 index: 254. Preceding OpId from leader: term: 2 index: 255. (index mismatch)
I20240502 01:58:49.872431 14595 consensus_queue.cc:1035] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c191817e0ce48b3b552af940f9d11ae" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44991 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 255, Last known committed idx: 254, Time since last communication: 0.000s
I20240502 01:58:49.875257 14179 raft_consensus.cc:1261] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15 [term 2 FOLLOWER]: Refusing update from remote peer 089e23168d0b48a8baf89ee203e1b4fd: Log matching property violated. Preceding OpId in replica: term: 1 index: 254. Preceding OpId from leader: term: 2 index: 255. (index mismatch)
I20240502 01:58:49.875710 14595 consensus_queue.cc:1035] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 255, Last known committed idx: 254, Time since last communication: 0.000s
I20240502 01:58:50.197094 14179 consensus_queue.cc:237] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 290, Committed index: 290, Last appended: 2.290, Last appended by leader: 289, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 291 OBSOLETE_local: false peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:50.198100 12861 raft_consensus.cc:1261] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 2 FOLLOWER]: Refusing update from remote peer c9900a57125149e69854e49405383e15: Log matching property violated. Preceding OpId in replica: term: 2 index: 290. Preceding OpId from leader: term: 2 index: 291. (index mismatch)
I20240502 01:58:50.198460 14693 consensus_queue.cc:1035] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 291, Last known committed idx: 290, Time since last communication: 0.000s
I20240502 01:58:50.200049 14699 raft_consensus.cc:2932] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 [term 2 LEADER]: Committing config change with OpId 2.291: config changed from index 289 to 291, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New config: { opid_index: 291 OBSOLETE_local: false peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:50.200443 12861 raft_consensus.cc:2932] T 1568709dd2424f188f484adba0c720d4 P 76bcefd6842e4c09ba16b8b6aa1ef662 [term 2 FOLLOWER]: Committing config change with OpId 2.291: config changed from index 289 to 291, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New config: { opid_index: 291 OBSOLETE_local: false peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:50.201516 11127 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1568709dd2424f188f484adba0c720d4 with cas_config_opid_index 289: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:50.202401 11133 catalog_manager.cc:5484] T 1568709dd2424f188f484adba0c720d4 P c9900a57125149e69854e49405383e15 reported cstate change: config changed from index 289 to 291, VOTER a47375810d964f128aae1e2df1370b79 (127.7.221.65) evicted. New cstate: current_term: 2 leader_uuid: "c9900a57125149e69854e49405383e15" committed_config { opid_index: 291 OBSOLETE_local: false peers { permanent_uuid: "76bcefd6842e4c09ba16b8b6aa1ef662" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 46849 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:58:50.204770 11266 tablet_service.cc:1514] Processing DeleteTablet for tablet 1568709dd2424f188f484adba0c720d4 with delete_type TABLET_DATA_TOMBSTONED (TS a47375810d964f128aae1e2df1370b79 not found in new config with opid_index 291) from {username='slave'} at 127.0.0.1:49072
I20240502 01:58:50.205174 14742 tablet_replica.cc:330] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79: stopping tablet replica
I20240502 01:58:50.205329 14742 raft_consensus.cc:2229] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:50.205452 14742 raft_consensus.cc:2258] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:50.206936 14742 ts_tablet_manager.cc:1875] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:50.208739 14742 ts_tablet_manager.cc:1888] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.290
I20240502 01:58:50.208819 14742 log.cc:1196] T 1568709dd2424f188f484adba0c720d4 P a47375810d964f128aae1e2df1370b79: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/1568709dd2424f188f484adba0c720d4
I20240502 01:58:50.209316 11127 catalog_manager.cc:4830] TS a47375810d964f128aae1e2df1370b79 (127.7.221.65:36857): tablet 1568709dd2424f188f484adba0c720d4 (table rebalance_test_table_4 [id=c6e32f615b8e4e1c937ef19c3ab2c37a]) successfully deleted
I20240502 01:58:50.320685 12589 consensus_queue.cc:237] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 255, Committed index: 255, Last appended: 2.255, Last appended by leader: 254, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 256 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } }
I20240502 01:58:50.321671 14179 raft_consensus.cc:1261] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15 [term 2 FOLLOWER]: Refusing update from remote peer 089e23168d0b48a8baf89ee203e1b4fd: Log matching property violated. Preceding OpId in replica: term: 2 index: 255. Preceding OpId from leader: term: 2 index: 256. (index mismatch)
I20240502 01:58:50.322042 14730 consensus_queue.cc:1035] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 256, Last known committed idx: 255, Time since last communication: 0.000s
I20240502 01:58:50.323588 14595 raft_consensus.cc:2932] T a1596ccad20249358ffbd0a4e3a5e1cb P 089e23168d0b48a8baf89ee203e1b4fd [term 2 LEADER]: Committing config change with OpId 2.256: config changed from index 254 to 256, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New config: { opid_index: 256 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:50.323985 14179 raft_consensus.cc:2932] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15 [term 2 FOLLOWER]: Committing config change with OpId 2.256: config changed from index 254 to 256, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New config: { opid_index: 256 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:50.325245 11128 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a1596ccad20249358ffbd0a4e3a5e1cb with cas_config_opid_index 254: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:58:50.326339 11139 catalog_manager.cc:5484] T a1596ccad20249358ffbd0a4e3a5e1cb P c9900a57125149e69854e49405383e15 reported cstate change: config changed from index 254 to 256, VOTER 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67) evicted. New cstate: current_term: 2 leader_uuid: "089e23168d0b48a8baf89ee203e1b4fd" committed_config { opid_index: 256 OBSOLETE_local: false peers { permanent_uuid: "089e23168d0b48a8baf89ee203e1b4fd" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 43337 } attrs { promote: false } } peers { permanent_uuid: "c9900a57125149e69854e49405383e15" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 38577 } attrs { promote: false } } }
I20240502 01:58:50.328502 11531 tablet_service.cc:1514] Processing DeleteTablet for tablet a1596ccad20249358ffbd0a4e3a5e1cb with delete_type TABLET_DATA_TOMBSTONED (TS 8c191817e0ce48b3b552af940f9d11ae not found in new config with opid_index 256) from {username='slave'} at 127.0.0.1:44872
I20240502 01:58:50.328848 14758 tablet_replica.cc:330] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae: stopping tablet replica
I20240502 01:58:50.328972 14758 raft_consensus.cc:2229] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 01:58:50.329072 14758 raft_consensus.cc:2258] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 01:58:50.330255 14758 ts_tablet_manager.cc:1875] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:58:50.332165 14758 ts_tablet_manager.cc:1888] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.255
I20240502 01:58:50.332237 14758 log.cc:1196] T a1596ccad20249358ffbd0a4e3a5e1cb P 8c191817e0ce48b3b552af940f9d11ae: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/a1596ccad20249358ffbd0a4e3a5e1cb
I20240502 01:58:50.332722 11128 catalog_manager.cc:4830] TS 8c191817e0ce48b3b552af940f9d11ae (127.7.221.67:44991): tablet a1596ccad20249358ffbd0a4e3a5e1cb (table rebalance_test_table_0 [id=d73cd318a885440582b140642ef6635e]) successfully deleted
I20240502 01:58:50.559585 12568 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 01:58:50.560964 12840 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:50.561820 12704 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:50.569605 12978 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:50.571007 14159 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 01:58:50.582124 11266 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:50.587487 11399 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 01:58:50.601908 11531 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 01:58:50.675022 12704 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:50.677098 11399 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 01:58:50.679060 12840 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:50.681308 12568 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 01:58:50.683621 11531 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 01:58:50.695910 11266 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:50.707937 14159 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 01:58:50.711323 12978 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:50.797027 12704 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:58:50.800731 12840 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:50.802248 12568 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 01:58:50.806046 11266 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 01:58:50.808810 11399 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 01:58:50.818526 11531 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 01:58:50.819739 12978 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:58:50.842886 14159 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
629fef51a73345408bea08d1a5ecc913 | 127.7.221.126:32951 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
allow_unsafe_replication_factor | true | runtime,unsafe | all 1 server(s) checked
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.7.221.84:39105 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+---------+----------+----------------+-----------------
0190cccef0ae48f3baa4123c0e4b49b0 | 127.7.221.69:35979 | HEALTHY | <none> | 0 | 0
089e23168d0b48a8baf89ee203e1b4fd | 127.7.221.68:43337 | HEALTHY | <none> | 2 | 0
6bf256be71684cdf87ef861e5890b614 | 127.7.221.66:36035 | HEALTHY | <none> | 3 | 0
76bcefd6842e4c09ba16b8b6aa1ef662 | 127.7.221.70:46849 | HEALTHY | <none> | 1 | 0
8c191817e0ce48b3b552af940f9d11ae | 127.7.221.67:44991 | HEALTHY | <none> | 3 | 0
a47375810d964f128aae1e2df1370b79 | 127.7.221.65:36857 | HEALTHY | <none> | 4 | 0
c9900a57125149e69854e49405383e15 | 127.7.221.72:38577 | HEALTHY | <none> | 2 | 0
f271c45722d545f7b33fe7f407444dcb | 127.7.221.71:38753 | HEALTHY | <none> | 1 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 8
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 8 server(s) checked
local_ip_for_outbound_sockets | 127.7.221.65 | experimental | 127.7.221.65:36857
local_ip_for_outbound_sockets | 127.7.221.66 | experimental | 127.7.221.66:36035
local_ip_for_outbound_sockets | 127.7.221.67 | experimental | 127.7.221.67:44991
local_ip_for_outbound_sockets | 127.7.221.68 | experimental | 127.7.221.68:43337
local_ip_for_outbound_sockets | 127.7.221.69 | experimental | 127.7.221.69:35979
local_ip_for_outbound_sockets | 127.7.221.70 | experimental | 127.7.221.70:46849
local_ip_for_outbound_sockets | 127.7.221.71 | experimental | 127.7.221.71:38753
local_ip_for_outbound_sockets | 127.7.221.72 | experimental | 127.7.221.72:38577
never_fsync | true | unsafe,advanced | all 8 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 8 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 8 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 8 server(s) checked
server_dump_info_format | pb | hidden | all 8 server(s) checked
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.7.221.65:36857
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.7.221.66:36035
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.7.221.67:44991
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.7.221.68:43337
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.7.221.69:35979
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.7.221.70:46849
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/info.pb | hidden | 127.7.221.71:38753
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalanceParamTest.Rebalance_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data/info.pb | hidden | 127.7.221.72:38577
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.7.221.84:39105 | all 8 server(s) checked
time_source | builtin | all 8 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 9 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
TestTable | 2 | HEALTHY | 1 | 1 | 0 | 0 | 0
rebalance_test_table_0 | 2 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 2 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 2 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 2 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_4 | 2 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 4
First Quartile | 4
Median | 4
Third Quartile | 4
Maximum | 4
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 8
Tables | 6
Tablets | 16
Replicas | 32
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240502 01:58:50.860857 8053 log_verifier.cc:126] Checking tablet 055b7173c3174a019457ad258989238a
I20240502 01:58:50.911688 8053 log_verifier.cc:177] Verified matching terms for 276 ops in tablet 055b7173c3174a019457ad258989238a
I20240502 01:58:50.911810 8053 log_verifier.cc:126] Checking tablet 0f1917ed678a4b8798e917d9c4e511da
I20240502 01:58:50.956930 8053 log_verifier.cc:177] Verified matching terms for 272 ops in tablet 0f1917ed678a4b8798e917d9c4e511da
I20240502 01:58:50.957033 8053 log_verifier.cc:126] Checking tablet 1568709dd2424f188f484adba0c720d4
I20240502 01:58:51.006474 8053 log_verifier.cc:177] Verified matching terms for 291 ops in tablet 1568709dd2424f188f484adba0c720d4
I20240502 01:58:51.006636 8053 log_verifier.cc:126] Checking tablet 1819f2f9c847491886d78211e3ca0fcc
I20240502 01:58:51.015353 8053 log_verifier.cc:177] Verified matching terms for 2 ops in tablet 1819f2f9c847491886d78211e3ca0fcc
I20240502 01:58:51.015408 8053 log_verifier.cc:126] Checking tablet 2dd97f65baa14cd9b3010b5e20592f1f
I20240502 01:58:51.057924 8053 log_verifier.cc:177] Verified matching terms for 237 ops in tablet 2dd97f65baa14cd9b3010b5e20592f1f
I20240502 01:58:51.058069 8053 log_verifier.cc:126] Checking tablet 495cddfb76b241a89676a723a9cc8b44
I20240502 01:58:51.103624 8053 log_verifier.cc:177] Verified matching terms for 277 ops in tablet 495cddfb76b241a89676a723a9cc8b44
I20240502 01:58:51.103736 8053 log_verifier.cc:126] Checking tablet 4ba576fba5b44e149c1c6a302248fd0f
I20240502 01:58:51.144870 8053 log_verifier.cc:177] Verified matching terms for 248 ops in tablet 4ba576fba5b44e149c1c6a302248fd0f
I20240502 01:58:51.144977 8053 log_verifier.cc:126] Checking tablet 4cef15d5c5fc41cbb55d184731eff9e2
I20240502 01:58:51.188761 8053 log_verifier.cc:177] Verified matching terms for 261 ops in tablet 4cef15d5c5fc41cbb55d184731eff9e2
I20240502 01:58:51.188872 8053 log_verifier.cc:126] Checking tablet 5ddde4eecc19461db50d15139066ea31
I20240502 01:58:51.234094 8053 log_verifier.cc:177] Verified matching terms for 280 ops in tablet 5ddde4eecc19461db50d15139066ea31
I20240502 01:58:51.234233 8053 log_verifier.cc:126] Checking tablet 706c5ef6b1b24ce28a70cef754b74d73
I20240502 01:58:51.272186 8053 log_verifier.cc:177] Verified matching terms for 213 ops in tablet 706c5ef6b1b24ce28a70cef754b74d73
I20240502 01:58:51.272317 8053 log_verifier.cc:126] Checking tablet 91e43d66f1664c7db8f1f16f623640be
I20240502 01:58:51.324368 8053 log_verifier.cc:177] Verified matching terms for 256 ops in tablet 91e43d66f1664c7db8f1f16f623640be
I20240502 01:58:51.324520 8053 log_verifier.cc:126] Checking tablet 9375f41e0aad43d3a9d6f5ef7ac73880
I20240502 01:58:51.375810 8053 log_verifier.cc:177] Verified matching terms for 287 ops in tablet 9375f41e0aad43d3a9d6f5ef7ac73880
I20240502 01:58:51.376085 8053 log_verifier.cc:126] Checking tablet a1596ccad20249358ffbd0a4e3a5e1cb
I20240502 01:58:51.421276 8053 log_verifier.cc:177] Verified matching terms for 256 ops in tablet a1596ccad20249358ffbd0a4e3a5e1cb
I20240502 01:58:51.421394 8053 log_verifier.cc:126] Checking tablet a408622a1ff546a6b70bb23101521b90
I20240502 01:58:51.470005 8053 log_verifier.cc:177] Verified matching terms for 283 ops in tablet a408622a1ff546a6b70bb23101521b90
I20240502 01:58:51.470162 8053 log_verifier.cc:126] Checking tablet a6b4f69afc6149feac9ca1e82ae995ff
I20240502 01:58:51.513352 8053 log_verifier.cc:177] Verified matching terms for 251 ops in tablet a6b4f69afc6149feac9ca1e82ae995ff
I20240502 01:58:51.513468 8053 log_verifier.cc:126] Checking tablet e33cb78a2b63498f9c9485b8f751859e
I20240502 01:58:51.558553 8053 log_verifier.cc:177] Verified matching terms for 261 ops in tablet e33cb78a2b63498f9c9485b8f751859e
I20240502 01:58:51.560189 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 11203
I20240502 01:58:51.569346 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 11336
I20240502 01:58:51.576340 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 11469
I20240502 01:58:51.583227 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 12504
I20240502 01:58:51.589341 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 12640
I20240502 01:58:51.595325 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 12774
I20240502 01:58:51.601296 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 12912
I20240502 01:58:51.607275 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 14095
I20240502 01:58:51.613101 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 11111
2024-05-02T01:58:51Z chronyd exiting
[ OK ] RebalanceParamTest.Rebalance/3 (23540 ms)
[----------] 1 test from RebalanceParamTest (23540 ms total)
[----------] 1 test from ConcurrentRebalancersTest
[ RUN ] ConcurrentRebalancersTest.TwoConcurrentRebalancers/1
I20240502 01:58:51.668329 8053 test_util.cc:268] Using random seed: -237404319
I20240502 01:58:51.669142 8053 ts_itest-base.cc:112] Starting cluster with:
I20240502 01:58:51.669178 8053 ts_itest-base.cc:113] --------------
I20240502 01:58:51.669189 8053 ts_itest-base.cc:114] 8 tablet servers
I20240502 01:58:51.669200 8053 ts_itest-base.cc:115] 3 replicas per TS
I20240502 01:58:51.669215 8053 ts_itest-base.cc:116] --------------
2024-05-02T01:58:51Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T01:58:51Z Disabled control of system clock
I20240502 01:58:51.677403 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.7.221.126:37149
--webserver_interface=127.7.221.126
--webserver_port=0
--builtin_ntp_servers=127.7.221.84:36353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.7.221.126:37149
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false with env {}
W20240502 01:58:51.781247 14883 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:51.781435 14883 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:51.781476 14883 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:51.784351 14883 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:51.784404 14883 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 01:58:51.784427 14883 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:51.784451 14883 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 01:58:51.784469 14883 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 01:58:51.787868 14883 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.7.221.126:37149
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.7.221.126:37149
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.7.221.126
--webserver_port=0
--never_fsync=true
--heap_profile_path=/tmp/kudu.14883
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:51.788583 14883 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:51.789556 14883 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:51.796808 14891 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:51.797608 14888 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:51.798301 14889 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:51.799106 14883 server_base.cc:1034] running on GCE node
I20240502 01:58:51.799511 14883 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:51.800312 14883 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:51.801463 14883 hybrid_clock.cc:648] HybridClock initialized: now 1714615131801439 us; error 36 us; skew 500 ppm
I20240502 01:58:51.803270 14883 webserver.cc:460] Webserver started at http://127.7.221.126:34065/ using document root <none> and password file <none>
I20240502 01:58:51.803797 14883 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:51.803854 14883 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:51.804001 14883 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:51.805502 14883 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "0d981b1e50234e83abbb17940d1327fd"
format_stamp: "Formatted at 2024-05-02 01:58:51 on dist-test-slave-dmxt"
I20240502 01:58:51.805917 14883 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "0d981b1e50234e83abbb17940d1327fd"
format_stamp: "Formatted at 2024-05-02 01:58:51 on dist-test-slave-dmxt"
I20240502 01:58:51.809003 14883 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.001s sys 0.004s
I20240502 01:58:51.811014 14897 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:51.811945 14883 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:51.812052 14883 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
uuid: "0d981b1e50234e83abbb17940d1327fd"
format_stamp: "Formatted at 2024-05-02 01:58:51 on dist-test-slave-dmxt"
I20240502 01:58:51.812121 14883 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:51.823906 14883 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:51.824433 14883 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:51.824553 14883 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:51.830816 14883 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.126:37149
I20240502 01:58:51.830822 14951 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.126:37149 every 8 connection(s)
I20240502 01:58:51.831660 14883 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 01:58:51.831960 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 14883
I20240502 01:58:51.832085 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 01:58:51.834372 14952 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:51.839766 14952 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd: Bootstrap starting.
I20240502 01:58:51.841905 14952 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:51.842581 14952 log.cc:824] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:51.844146 14952 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd: No bootstrap required, opened a new log
I20240502 01:58:51.846443 14952 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0d981b1e50234e83abbb17940d1327fd" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 37149 } }
I20240502 01:58:51.846637 14952 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:51.846700 14952 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0d981b1e50234e83abbb17940d1327fd, State: Initialized, Role: FOLLOWER
I20240502 01:58:51.847122 14952 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0d981b1e50234e83abbb17940d1327fd" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 37149 } }
I20240502 01:58:51.847268 14952 raft_consensus.cc:388] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 01:58:51.847375 14952 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 01:58:51.847503 14952 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:51.848191 14952 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0d981b1e50234e83abbb17940d1327fd" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 37149 } }
I20240502 01:58:51.848486 14952 leader_election.cc:302] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 0d981b1e50234e83abbb17940d1327fd; no voters:
I20240502 01:58:51.848729 14952 leader_election.cc:288] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 01:58:51.848811 14957 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:51.849025 14957 raft_consensus.cc:686] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [term 1 LEADER]: Becoming Leader. State: Replica: 0d981b1e50234e83abbb17940d1327fd, State: Running, Role: LEADER
I20240502 01:58:51.849570 14957 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0d981b1e50234e83abbb17940d1327fd" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 37149 } }
I20240502 01:58:51.849728 14952 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [sys.catalog]: configured and running, proceeding with master startup.
I20240502 01:58:51.850911 14958 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "0d981b1e50234e83abbb17940d1327fd" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0d981b1e50234e83abbb17940d1327fd" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 37149 } } }
I20240502 01:58:51.851047 14958 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [sys.catalog]: This master's current role is: LEADER
I20240502 01:58:51.851413 14966 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 01:58:51.851372 14959 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0d981b1e50234e83abbb17940d1327fd. Latest consensus state: current_term: 1 leader_uuid: "0d981b1e50234e83abbb17940d1327fd" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0d981b1e50234e83abbb17940d1327fd" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 37149 } } }
I20240502 01:58:51.851466 14959 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd [sys.catalog]: This master's current role is: LEADER
I20240502 01:58:51.854166 14966 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 01:58:51.858879 14966 catalog_manager.cc:1347] Generated new cluster ID: d5643d04cb2943df991e6632f148e6cd
I20240502 01:58:51.858947 14966 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 01:58:51.888679 14966 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 01:58:51.889557 14966 catalog_manager.cc:1504] Loading token signing keys...
I20240502 01:58:51.898535 14966 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 0d981b1e50234e83abbb17940d1327fd: Generated new TSK 0
I20240502 01:58:51.899302 14966 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 01:58:51.906136 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.65:0
--local_ip_for_outbound_sockets=127.7.221.65
--webserver_interface=127.7.221.65
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--builtin_ntp_servers=127.7.221.84:36353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 01:58:52.014215 14977 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:52.014452 14977 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:52.014530 14977 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:52.017658 14977 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:52.017733 14977 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:52.017849 14977 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.65
I20240502 01:58:52.021342 14977 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.7.221.65
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--never_fsync=true
--heap_profile_path=/tmp/kudu.14977
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.65
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:52.022229 14977 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:52.023327 14977 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:52.031744 14983 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:52.032560 14985 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:52.033242 14982 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:52.034107 14977 server_base.cc:1034] running on GCE node
I20240502 01:58:52.034417 14977 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:52.034948 14977 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:52.036100 14977 hybrid_clock.cc:648] HybridClock initialized: now 1714615132036090 us; error 24 us; skew 500 ppm
I20240502 01:58:52.037788 14977 webserver.cc:460] Webserver started at http://127.7.221.65:39945/ using document root <none> and password file <none>
I20240502 01:58:52.038285 14977 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:52.038339 14977 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:52.038497 14977 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:52.040052 14977 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "ddae2a636423489693c12b7d7d166c09"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.040498 14977 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "ddae2a636423489693c12b7d7d166c09"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.043569 14977 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.004s sys 0.000s
I20240502 01:58:52.045536 14991 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.046422 14977 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:52.046519 14977 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "ddae2a636423489693c12b7d7d166c09"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.046617 14977 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:52.074892 14977 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:52.075417 14977 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:52.075567 14977 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:52.076071 14977 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:52.076862 14977 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:52.076910 14977 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.076962 14977 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:52.076997 14977 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.084827 14977 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.65:46123
I20240502 01:58:52.084867 15107 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.65:46123 every 8 connection(s)
I20240502 01:58:52.085662 14977 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 01:58:52.091836 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 14977
I20240502 01:58:52.091961 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 01:58:52.093789 15108 heartbeater.cc:347] Connected to a master server at 127.7.221.126:37149
I20240502 01:58:52.093987 15108 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:52.094045 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.66:0
--local_ip_for_outbound_sockets=127.7.221.66
--webserver_interface=127.7.221.66
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--builtin_ntp_servers=127.7.221.84:36353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:52.094475 15108 heartbeater.cc:511] Master 127.7.221.126:37149 requested a full tablet report, sending...
I20240502 01:58:52.095897 14915 ts_manager.cc:194] Registered new tserver with Master: ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:58:52.097088 14915 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.65:53073
W20240502 01:58:52.195859 15112 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:52.196094 15112 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:52.196159 15112 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:52.200729 15112 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:52.200874 15112 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:52.201067 15112 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.66
I20240502 01:58:52.204756 15112 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.7.221.66
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--never_fsync=true
--heap_profile_path=/tmp/kudu.15112
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.66
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:52.205610 15112 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:52.206770 15112 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:52.215188 15117 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:52.215790 15120 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:52.216708 15118 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:52.217482 15112 server_base.cc:1034] running on GCE node
I20240502 01:58:52.217860 15112 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:52.218389 15112 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:52.219568 15112 hybrid_clock.cc:648] HybridClock initialized: now 1714615132219554 us; error 33 us; skew 500 ppm
I20240502 01:58:52.221381 15112 webserver.cc:460] Webserver started at http://127.7.221.66:40923/ using document root <none> and password file <none>
I20240502 01:58:52.221913 15112 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:52.221972 15112 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:52.222141 15112 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:52.223768 15112 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.224231 15112 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.227372 15112 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.004s sys 0.000s
I20240502 01:58:52.229526 15126 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.230463 15112 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:52.230569 15112 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.230679 15112 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:52.255618 15112 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:52.256218 15112 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:52.256381 15112 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:52.256908 15112 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:52.257773 15112 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:52.257823 15112 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.257882 15112 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:52.257915 15112 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.266031 15112 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.66:34905
I20240502 01:58:52.266079 15242 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.66:34905 every 8 connection(s)
I20240502 01:58:52.266943 15112 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 01:58:52.269881 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 15112
I20240502 01:58:52.270005 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 01:58:52.272012 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.67:0
--local_ip_for_outbound_sockets=127.7.221.67
--webserver_interface=127.7.221.67
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--builtin_ntp_servers=127.7.221.84:36353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:52.275578 15243 heartbeater.cc:347] Connected to a master server at 127.7.221.126:37149
I20240502 01:58:52.275794 15243 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:52.276283 15243 heartbeater.cc:511] Master 127.7.221.126:37149 requested a full tablet report, sending...
I20240502 01:58:52.277323 14915 ts_manager.cc:194] Registered new tserver with Master: 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:58:52.278162 14915 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.66:38687
W20240502 01:58:52.374366 15246 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:52.374568 15246 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:52.374658 15246 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:52.377493 15246 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:52.377548 15246 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:52.377609 15246 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.67
I20240502 01:58:52.380824 15246 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.7.221.67
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--never_fsync=true
--heap_profile_path=/tmp/kudu.15246
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.67
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:52.381541 15246 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:52.382505 15246 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:52.390801 15252 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:52.391474 15251 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:52.392048 15254 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:52.392882 15246 server_base.cc:1034] running on GCE node
I20240502 01:58:52.393249 15246 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:52.393766 15246 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:52.394917 15246 hybrid_clock.cc:648] HybridClock initialized: now 1714615132394891 us; error 42 us; skew 500 ppm
I20240502 01:58:52.396754 15246 webserver.cc:460] Webserver started at http://127.7.221.67:46411/ using document root <none> and password file <none>
I20240502 01:58:52.397267 15246 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:52.397329 15246 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:52.397492 15246 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:52.399163 15246 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "f77c2a969bde45b49ac1bbbe242ff171"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.399624 15246 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "f77c2a969bde45b49ac1bbbe242ff171"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.402797 15246 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 01:58:52.404846 15262 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.405787 15246 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:52.405902 15246 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "f77c2a969bde45b49ac1bbbe242ff171"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.405997 15246 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:52.417130 15246 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:52.417631 15246 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:52.417809 15246 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:52.418282 15246 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:52.419169 15246 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:52.419217 15246 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.419291 15246 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:52.419324 15246 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.427363 15246 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.67:44157
I20240502 01:58:52.427420 15376 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.67:44157 every 8 connection(s)
I20240502 01:58:52.428232 15246 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 01:58:52.436147 15377 heartbeater.cc:347] Connected to a master server at 127.7.221.126:37149
I20240502 01:58:52.436360 15377 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:52.436802 15377 heartbeater.cc:511] Master 127.7.221.126:37149 requested a full tablet report, sending...
I20240502 01:58:52.437486 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 15246
I20240502 01:58:52.437595 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 01:58:52.437713 14915 ts_manager.cc:194] Registered new tserver with Master: f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:58:52.438267 14915 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.67:43885
I20240502 01:58:52.439754 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.68:0
--local_ip_for_outbound_sockets=127.7.221.68
--webserver_interface=127.7.221.68
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--builtin_ntp_servers=127.7.221.84:36353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 01:58:52.537305 15380 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:52.537501 15380 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:52.537544 15380 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:52.540491 15380 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:52.540550 15380 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:52.540622 15380 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.68
I20240502 01:58:52.544096 15380 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.68:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.7.221.68
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--never_fsync=true
--heap_profile_path=/tmp/kudu.15380
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.68
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:52.544920 15380 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:52.545966 15380 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:52.554319 15385 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:52.555125 15386 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:52.555780 15388 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:52.556732 15380 server_base.cc:1034] running on GCE node
I20240502 01:58:52.557107 15380 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:52.557632 15380 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:52.558779 15380 hybrid_clock.cc:648] HybridClock initialized: now 1714615132558769 us; error 28 us; skew 500 ppm
I20240502 01:58:52.560541 15380 webserver.cc:460] Webserver started at http://127.7.221.68:37673/ using document root <none> and password file <none>
I20240502 01:58:52.561069 15380 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:52.561131 15380 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:52.561295 15380 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:52.562970 15380 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.563423 15380 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.566615 15380 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 01:58:52.568624 15395 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.569566 15380 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:52.569669 15380 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.569746 15380 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:52.584359 15380 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:52.584892 15380 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:52.585042 15380 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:52.585502 15380 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:52.586305 15380 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:52.586354 15380 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.586407 15380 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:52.586439 15380 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.594190 15380 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.68:34323
I20240502 01:58:52.594214 15515 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.68:34323 every 8 connection(s)
I20240502 01:58:52.595057 15380 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 01:58:52.602541 15516 heartbeater.cc:347] Connected to a master server at 127.7.221.126:37149
I20240502 01:58:52.602754 15516 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:52.603192 15516 heartbeater.cc:511] Master 127.7.221.126:37149 requested a full tablet report, sending...
I20240502 01:58:52.604048 14914 ts_manager.cc:194] Registered new tserver with Master: 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:58:52.604629 14914 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.68:33357
I20240502 01:58:52.605090 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 15380
I20240502 01:58:52.605221 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 01:58:52.607129 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.69:0
--local_ip_for_outbound_sockets=127.7.221.69
--webserver_interface=127.7.221.69
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--builtin_ntp_servers=127.7.221.84:36353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 01:58:52.706169 15519 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:52.706344 15519 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:52.706400 15519 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:52.709225 15519 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:52.709280 15519 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:52.709368 15519 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.69
I20240502 01:58:52.712674 15519 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.69:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.7.221.69
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--never_fsync=true
--heap_profile_path=/tmp/kudu.15519
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.69
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:52.713425 15519 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:52.714395 15519 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:52.722303 15524 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:52.723187 15527 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:52.723872 15525 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:52.724792 15519 server_base.cc:1034] running on GCE node
I20240502 01:58:52.725162 15519 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:52.725632 15519 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:52.726776 15519 hybrid_clock.cc:648] HybridClock initialized: now 1714615132726762 us; error 31 us; skew 500 ppm
I20240502 01:58:52.728416 15519 webserver.cc:460] Webserver started at http://127.7.221.69:41583/ using document root <none> and password file <none>
I20240502 01:58:52.728891 15519 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:52.728950 15519 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:52.729127 15519 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:52.730731 15519 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "84987ad9e4a54d96af8569e4aaf0ed64"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.731184 15519 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "84987ad9e4a54d96af8569e4aaf0ed64"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.734099 15519 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:52.736224 15533 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.737170 15519 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.001s
I20240502 01:58:52.737267 15519 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "84987ad9e4a54d96af8569e4aaf0ed64"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.737340 15519 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:52.766381 15519 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:52.766997 15519 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:52.767159 15519 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:52.767648 15519 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:52.768520 15519 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:52.768568 15519 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.768630 15519 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:52.768654 15519 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.777050 15519 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.69:35223
I20240502 01:58:52.777079 15652 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.69:35223 every 8 connection(s)
I20240502 01:58:52.777935 15519 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 01:58:52.782783 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 15519
I20240502 01:58:52.782909 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 01:58:52.785049 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.70:0
--local_ip_for_outbound_sockets=127.7.221.70
--webserver_interface=127.7.221.70
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--builtin_ntp_servers=127.7.221.84:36353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:52.786535 15653 heartbeater.cc:347] Connected to a master server at 127.7.221.126:37149
I20240502 01:58:52.786828 15653 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:52.787427 15653 heartbeater.cc:511] Master 127.7.221.126:37149 requested a full tablet report, sending...
I20240502 01:58:52.788409 14914 ts_manager.cc:194] Registered new tserver with Master: 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223)
I20240502 01:58:52.789217 14914 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.69:44735
W20240502 01:58:52.884368 15656 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:52.884560 15656 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:52.884600 15656 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:52.887315 15656 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:52.887363 15656 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:52.887419 15656 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.70
I20240502 01:58:52.890460 15656 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.70:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.7.221.70
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--never_fsync=true
--heap_profile_path=/tmp/kudu.15656
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.70
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:52.891196 15656 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:52.892104 15656 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:52.900384 15665 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:52.901115 15661 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:52.901881 15662 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:52.902774 15656 server_base.cc:1034] running on GCE node
I20240502 01:58:52.903152 15656 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:52.903640 15656 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:52.904790 15656 hybrid_clock.cc:648] HybridClock initialized: now 1714615132904783 us; error 24 us; skew 500 ppm
I20240502 01:58:52.906662 15656 webserver.cc:460] Webserver started at http://127.7.221.70:33431/ using document root <none> and password file <none>
I20240502 01:58:52.907195 15656 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:52.907254 15656 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:52.907411 15656 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:52.908933 15656 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/instance:
uuid: "d4e5422f66454511ae5a50ec6ab134ad"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.909353 15656 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal/instance:
uuid: "d4e5422f66454511ae5a50ec6ab134ad"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.912418 15656 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 01:58:52.914419 15671 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.915460 15656 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:52.915578 15656 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "d4e5422f66454511ae5a50ec6ab134ad"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:58:52.915661 15656 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:52.930506 15656 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:52.931041 15656 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:52.931161 15656 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:52.931569 15656 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:52.932322 15656 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:52.932365 15656 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.932399 15656 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:52.932417 15656 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:52.940181 15656 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.70:34649
I20240502 01:58:52.940208 15789 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.70:34649 every 8 connection(s)
I20240502 01:58:52.941047 15656 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 01:58:52.948910 15790 heartbeater.cc:347] Connected to a master server at 127.7.221.126:37149
I20240502 01:58:52.949122 15790 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:52.949579 15790 heartbeater.cc:511] Master 127.7.221.126:37149 requested a full tablet report, sending...
I20240502 01:58:52.950500 14914 ts_manager.cc:194] Registered new tserver with Master: d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649)
I20240502 01:58:52.950856 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 15656
I20240502 01:58:52.951001 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240502 01:58:52.951251 14914 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.70:52741
I20240502 01:58:52.953074 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.71:0
--local_ip_for_outbound_sockets=127.7.221.71
--webserver_interface=127.7.221.71
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--builtin_ntp_servers=127.7.221.84:36353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 01:58:53.053566 15793 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:53.053766 15793 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:53.053807 15793 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:53.056550 15793 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:53.056600 15793 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:53.056671 15793 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.71
I20240502 01:58:53.059818 15793 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.71:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.7.221.71
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--never_fsync=true
--heap_profile_path=/tmp/kudu.15793
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.71
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:53.060508 15793 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:53.061520 15793 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:53.069451 15799 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:53.070168 15798 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:53.070977 15801 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:53.071815 15793 server_base.cc:1034] running on GCE node
I20240502 01:58:53.072183 15793 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:53.072708 15793 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:53.073855 15793 hybrid_clock.cc:648] HybridClock initialized: now 1714615133073841 us; error 28 us; skew 500 ppm
I20240502 01:58:53.075573 15793 webserver.cc:460] Webserver started at http://127.7.221.71:43637/ using document root <none> and password file <none>
I20240502 01:58:53.076067 15793 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:53.076117 15793 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:53.076254 15793 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:53.077683 15793 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/instance:
uuid: "fc067ee690bb457e8109ad8e4eb43ab8"
format_stamp: "Formatted at 2024-05-02 01:58:53 on dist-test-slave-dmxt"
I20240502 01:58:53.078084 15793 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal/instance:
uuid: "fc067ee690bb457e8109ad8e4eb43ab8"
format_stamp: "Formatted at 2024-05-02 01:58:53 on dist-test-slave-dmxt"
I20240502 01:58:53.081020 15793 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 01:58:53.082917 15807 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:53.083799 15793 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:53.083889 15793 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "fc067ee690bb457e8109ad8e4eb43ab8"
format_stamp: "Formatted at 2024-05-02 01:58:53 on dist-test-slave-dmxt"
I20240502 01:58:53.083969 15793 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:53.093717 15793 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:53.094174 15793 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:53.094300 15793 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:53.094763 15793 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:53.095556 15793 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:53.095602 15793 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:53.095645 15793 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:53.095678 15793 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:53.099165 15108 heartbeater.cc:503] Master 127.7.221.126:37149 was elected leader, sending a full tablet report...
I20240502 01:58:53.103965 15793 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.71:43207
I20240502 01:58:53.103991 15922 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.71:43207 every 8 connection(s)
I20240502 01:58:53.104773 15793 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 01:58:53.108167 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 15793
I20240502 01:58:53.108295 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal/instance
I20240502 01:58:53.110292 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.72:0
--local_ip_for_outbound_sockets=127.7.221.72
--webserver_interface=127.7.221.72
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--builtin_ntp_servers=127.7.221.84:36353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:58:53.113083 15923 heartbeater.cc:347] Connected to a master server at 127.7.221.126:37149
I20240502 01:58:53.113263 15923 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:53.113685 15923 heartbeater.cc:511] Master 127.7.221.126:37149 requested a full tablet report, sending...
I20240502 01:58:53.114646 14914 ts_manager.cc:194] Registered new tserver with Master: fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207)
I20240502 01:58:53.115432 14914 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.71:35281
W20240502 01:58:53.212141 15926 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:53.212334 15926 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:58:53.212373 15926 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:53.215181 15926 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:58:53.215235 15926 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:58:53.215330 15926 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.72
I20240502 01:58:53.218657 15926 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.72:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.7.221.72
--webserver_port=0
--tserver_master_addrs=127.7.221.126:37149
--never_fsync=true
--heap_profile_path=/tmp/kudu.15926
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.72
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:58:53.219378 15926 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:58:53.220393 15926 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:58:53.228595 15931 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:53.229184 15932 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:58:53.230116 15934 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:58:53.230850 15926 server_base.cc:1034] running on GCE node
I20240502 01:58:53.231171 15926 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:58:53.231645 15926 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:58:53.232802 15926 hybrid_clock.cc:648] HybridClock initialized: now 1714615133232786 us; error 36 us; skew 500 ppm
I20240502 01:58:53.234666 15926 webserver.cc:460] Webserver started at http://127.7.221.72:41171/ using document root <none> and password file <none>
I20240502 01:58:53.235257 15926 fs_manager.cc:359] Metadata directory not provided
I20240502 01:58:53.235322 15926 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:58:53.235496 15926 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 01:58:53.237056 15926 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data/instance:
uuid: "3b8d615c71e44ff887e0f037c3efcff7"
format_stamp: "Formatted at 2024-05-02 01:58:53 on dist-test-slave-dmxt"
I20240502 01:58:53.237500 15926 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal/instance:
uuid: "3b8d615c71e44ff887e0f037c3efcff7"
format_stamp: "Formatted at 2024-05-02 01:58:53 on dist-test-slave-dmxt"
I20240502 01:58:53.240629 15926 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.001s sys 0.004s
I20240502 01:58:53.242579 15940 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:53.243536 15926 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:53.243639 15926 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "3b8d615c71e44ff887e0f037c3efcff7"
format_stamp: "Formatted at 2024-05-02 01:58:53 on dist-test-slave-dmxt"
I20240502 01:58:53.243716 15926 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:58:53.259248 15926 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:58:53.259822 15926 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:58:53.259976 15926 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:58:53.260485 15926 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:58:53.261343 15926 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:58:53.261420 15926 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:53.261483 15926 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:58:53.261512 15926 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:58:53.270514 15926 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.72:37645
I20240502 01:58:53.270547 16057 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.72:37645 every 8 connection(s)
I20240502 01:58:53.271430 15926 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 01:58:53.276113 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 15926
I20240502 01:58:53.276261 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal/instance
I20240502 01:58:53.279939 16058 heartbeater.cc:347] Connected to a master server at 127.7.221.126:37149
I20240502 01:58:53.280061 15243 heartbeater.cc:503] Master 127.7.221.126:37149 was elected leader, sending a full tablet report...
I20240502 01:58:53.280146 16058 heartbeater.cc:464] Registering TS with master...
I20240502 01:58:53.280674 16058 heartbeater.cc:511] Master 127.7.221.126:37149 requested a full tablet report, sending...
I20240502 01:58:53.281566 14914 ts_manager.cc:194] Registered new tserver with Master: 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 01:58:53.282049 14914 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.72:38853
I20240502 01:58:53.290118 8053 external_mini_cluster.cc:930] 8 TS(s) registered with all masters
I20240502 01:58:53.299085 8053 external_mini_cluster.cc:930] 8 TS(s) registered with all masters
I20240502 01:58:53.299151 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 15519
I20240502 01:58:53.306084 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 15656
I20240502 01:58:53.311645 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 15793
I20240502 01:58:53.317415 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 15926
I20240502 01:58:53.440240 15377 heartbeater.cc:503] Master 127.7.221.126:37149 was elected leader, sending a full tablet report...
I20240502 01:58:53.606582 15516 heartbeater.cc:503] Master 127.7.221.126:37149 was elected leader, sending a full tablet report...
I20240502 01:58:57.074379 14914 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54458:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:57.094460 15041 tablet_service.cc:1467] Processing CreateTablet for tablet e2f4ca025890492880608c7975650969 (DEFAULT_TABLE table=rebalance_test_table_0 [id=9736e7c5365849759564e386d208932f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.094461 15176 tablet_service.cc:1467] Processing CreateTablet for tablet 21e2e55d1a0d4b6eb18b4ade9468607f (DEFAULT_TABLE table=rebalance_test_table_0 [id=9736e7c5365849759564e386d208932f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.094456 15446 tablet_service.cc:1467] Processing CreateTablet for tablet 21e2e55d1a0d4b6eb18b4ade9468607f (DEFAULT_TABLE table=rebalance_test_table_0 [id=9736e7c5365849759564e386d208932f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.094732 15311 tablet_service.cc:1467] Processing CreateTablet for tablet 21e2e55d1a0d4b6eb18b4ade9468607f (DEFAULT_TABLE table=rebalance_test_table_0 [id=9736e7c5365849759564e386d208932f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.095431 15446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 21e2e55d1a0d4b6eb18b4ade9468607f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.095431 15311 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 21e2e55d1a0d4b6eb18b4ade9468607f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.095432 15176 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 21e2e55d1a0d4b6eb18b4ade9468607f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.095432 15041 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e2f4ca025890492880608c7975650969. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.096927 15310 tablet_service.cc:1467] Processing CreateTablet for tablet e2f4ca025890492880608c7975650969 (DEFAULT_TABLE table=rebalance_test_table_0 [id=9736e7c5365849759564e386d208932f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.097194 15310 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e2f4ca025890492880608c7975650969. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.098850 15040 tablet_service.cc:1467] Processing CreateTablet for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 (DEFAULT_TABLE table=rebalance_test_table_0 [id=9736e7c5365849759564e386d208932f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.098851 15175 tablet_service.cc:1467] Processing CreateTablet for tablet e2f4ca025890492880608c7975650969 (DEFAULT_TABLE table=rebalance_test_table_0 [id=9736e7c5365849759564e386d208932f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.098865 15445 tablet_service.cc:1467] Processing CreateTablet for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 (DEFAULT_TABLE table=rebalance_test_table_0 [id=9736e7c5365849759564e386d208932f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.099074 15040 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.099074 15175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e2f4ca025890492880608c7975650969. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.099095 15445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.100262 15174 tablet_service.cc:1467] Processing CreateTablet for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 (DEFAULT_TABLE table=rebalance_test_table_0 [id=9736e7c5365849759564e386d208932f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.100490 15174 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.103982 16079 tablet_bootstrap.cc:492] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:58:57.104583 16080 tablet_bootstrap.cc:492] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:58:57.106052 16079 tablet_bootstrap.cc:654] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.105348 16082 tablet_bootstrap.cc:492] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:58:57.106395 16080 tablet_bootstrap.cc:654] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.106909 16079 log.cc:824] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:57.107013 16080 log.cc:824] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:57.107545 16081 tablet_bootstrap.cc:492] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:58:57.107887 16082 tablet_bootstrap.cc:654] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.108814 16079 tablet_bootstrap.cc:492] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:58:57.108975 16079 ts_tablet_manager.cc:1367] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240502 01:58:57.109043 16082 log.cc:824] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:57.109285 16081 tablet_bootstrap.cc:654] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.109973 16081 log.cc:824] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09: Log is configured to *not* fsync() on all Append() calls
I20240502 01:58:57.110584 16082 tablet_bootstrap.cc:492] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:58:57.110584 16080 tablet_bootstrap.cc:492] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:58:57.110769 16082 ts_tablet_manager.cc:1367] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.006s user 0.004s sys 0.000s
I20240502 01:58:57.110781 16080 ts_tablet_manager.cc:1367] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.006s user 0.004s sys 0.000s
I20240502 01:58:57.111534 16081 tablet_bootstrap.cc:492] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:58:57.111685 16081 ts_tablet_manager.cc:1367] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 01:58:57.112555 16079 raft_consensus.cc:348] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.112751 16079 raft_consensus.cc:374] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.112800 16079 raft_consensus.cc:729] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.113281 16079 consensus_queue.cc:260] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.113639 16082 raft_consensus.cc:348] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.113834 16082 raft_consensus.cc:374] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.113890 16082 raft_consensus.cc:729] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.114133 16079 ts_tablet_manager.cc:1398] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s
I20240502 01:58:57.114452 16079 tablet_bootstrap.cc:492] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:58:57.114476 16080 raft_consensus.cc:348] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.114781 16080 raft_consensus.cc:374] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.114917 16080 raft_consensus.cc:729] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.115058 16082 consensus_queue.cc:260] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.115296 16079 tablet_bootstrap.cc:654] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.115458 16080 consensus_queue.cc:260] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.115937 16082 ts_tablet_manager.cc:1398] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.005s user 0.004s sys 0.000s
I20240502 01:58:57.116390 16079 tablet_bootstrap.cc:492] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:58:57.116483 16080 ts_tablet_manager.cc:1398] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.006s user 0.005s sys 0.000s
I20240502 01:58:57.116580 16079 ts_tablet_manager.cc:1367] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:57.117079 16079 raft_consensus.cc:348] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.117175 16079 raft_consensus.cc:374] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.117211 16079 raft_consensus.cc:729] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.117331 16079 consensus_queue.cc:260] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.117645 16079 ts_tablet_manager.cc:1398] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:57.117668 16080 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:58:57.118167 16081 raft_consensus.cc:348] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.118340 16081 raft_consensus.cc:374] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.118398 16081 raft_consensus.cc:729] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.118731 16080 tablet_bootstrap.cc:654] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.119196 16081 consensus_queue.cc:260] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.119369 16082 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:58:57.119908 16081 ts_tablet_manager.cc:1398] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.008s user 0.005s sys 0.000s
I20240502 01:58:57.119995 16080 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:58:57.120079 16080 ts_tablet_manager.cc:1367] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.001s
I20240502 01:58:57.120180 16082 tablet_bootstrap.cc:654] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.120195 16081 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:58:57.120589 16080 raft_consensus.cc:348] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.120684 16080 raft_consensus.cc:374] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.120715 16080 raft_consensus.cc:729] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.120824 16080 consensus_queue.cc:260] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.121096 16081 tablet_bootstrap.cc:654] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.121098 16080 ts_tablet_manager.cc:1398] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:57.122212 16082 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:58:57.122299 16082 ts_tablet_manager.cc:1367] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:57.122794 16082 raft_consensus.cc:348] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.122889 16082 raft_consensus.cc:374] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.122926 16082 raft_consensus.cc:729] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.123066 16082 consensus_queue.cc:260] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.123135 16081 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:58:57.123203 16081 ts_tablet_manager.cc:1367] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:57.123342 16082 ts_tablet_manager.cc:1398] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:57.123476 16082 tablet_bootstrap.cc:492] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:58:57.123680 16081 raft_consensus.cc:348] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.123777 16081 raft_consensus.cc:374] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.123831 16081 raft_consensus.cc:729] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.123944 16081 consensus_queue.cc:260] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.124274 16082 tablet_bootstrap.cc:654] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.124194 16081 ts_tablet_manager.cc:1398] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:57.125334 16082 tablet_bootstrap.cc:492] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:58:57.125422 16082 ts_tablet_manager.cc:1367] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:57.125885 16082 raft_consensus.cc:348] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.125981 16082 raft_consensus.cc:374] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.126019 16082 raft_consensus.cc:729] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.126161 16082 consensus_queue.cc:260] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.126488 16082 ts_tablet_manager.cc:1398] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:57.181397 16089 raft_consensus.cc:482] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:57.181577 16089 raft_consensus.cc:504] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.182624 16089 leader_election.cc:288] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
W20240502 01:58:57.183758 15378 tablet.cc:2369] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 01:58:57.187305 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2f4ca025890492880608c7975650969" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" is_pre_election: true
I20240502 01:58:57.187498 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2f4ca025890492880608c7975650969" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09" is_pre_election: true
I20240502 01:58:57.187602 15197 raft_consensus.cc:2454] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 0.
I20240502 01:58:57.187755 15061 raft_consensus.cc:2454] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 0.
I20240502 01:58:57.188076 15264 leader_election.cc:302] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:57.188303 16089 raft_consensus.cc:2781] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:57.188373 16089 raft_consensus.cc:482] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:57.188417 16089 raft_consensus.cc:3037] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:57.189253 16089 raft_consensus.cc:504] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.189543 16089 leader_election.cc:288] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 election: Requested vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:58:57.189864 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2f4ca025890492880608c7975650969" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:58:57.189909 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2f4ca025890492880608c7975650969" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:58:57.189972 15061 raft_consensus.cc:3037] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:57.190029 15197 raft_consensus.cc:3037] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:57.190851 15061 raft_consensus.cc:2454] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 1.
I20240502 01:58:57.191139 15265 leader_election.cc:302] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:57.191241 15197 raft_consensus.cc:2454] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 1.
I20240502 01:58:57.191329 16089 raft_consensus.cc:2781] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:57.191592 16089 raft_consensus.cc:686] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Becoming Leader. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Running, Role: LEADER
I20240502 01:58:57.191886 16089 consensus_queue.cc:237] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.194713 14913 catalog_manager.cc:5484] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: term changed from 0 to 1, leader changed from <none> to f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67). New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } }
I20240502 01:58:57.243304 16088 raft_consensus.cc:482] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:57.243443 16088 raft_consensus.cc:504] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.244493 16088 leader_election.cc:288] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:58:57.248977 15331 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "21e2e55d1a0d4b6eb18b4ade9468607f" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:58:57.249009 15466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "21e2e55d1a0d4b6eb18b4ade9468607f" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:58:57.249168 15331 raft_consensus.cc:2454] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:58:57.249276 15466 raft_consensus.cc:2454] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:58:57.249540 15129 leader_election.cc:302] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:57.249812 16088 raft_consensus.cc:2781] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:57.249872 16088 raft_consensus.cc:482] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:57.249898 16088 raft_consensus.cc:3037] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:57.250789 16088 raft_consensus.cc:504] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.251112 16088 leader_election.cc:288] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Requested vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:58:57.251446 15466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "21e2e55d1a0d4b6eb18b4ade9468607f" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:58:57.251509 15331 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "21e2e55d1a0d4b6eb18b4ade9468607f" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:58:57.251551 15466 raft_consensus.cc:3037] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:57.251629 15331 raft_consensus.cc:3037] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:57.252424 15331 raft_consensus.cc:2454] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:58:57.252444 15466 raft_consensus.cc:2454] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:58:57.252682 15129 leader_election.cc:302] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:57.252831 16088 raft_consensus.cc:2781] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:57.253022 16088 raft_consensus.cc:686] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Becoming Leader. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Running, Role: LEADER
I20240502 01:58:57.253327 16088 consensus_queue.cc:237] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.256080 14913 catalog_manager.cc:5484] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: term changed from 0 to 1, leader changed from <none> to 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66). New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:57.271364 15244 tablet.cc:2369] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 01:58:57.340749 15110 tablet.cc:2369] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 01:58:57.340735 16087 raft_consensus.cc:482] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:57.340895 16087 raft_consensus.cc:504] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.341953 16087 leader_election.cc:288] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:58:57.345711 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fcfc8dc4b90a4a6aa7e4035bbfdb4532" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09" is_pre_election: true
I20240502 01:58:57.345767 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fcfc8dc4b90a4a6aa7e4035bbfdb4532" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" is_pre_election: true
I20240502 01:58:57.345849 15061 raft_consensus.cc:2454] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 0.
I20240502 01:58:57.345906 15197 raft_consensus.cc:2454] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 0.
I20240502 01:58:57.346194 15398 leader_election.cc:302] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5, ddae2a636423489693c12b7d7d166c09; no voters:
I20240502 01:58:57.346480 16087 raft_consensus.cc:2781] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:57.346541 16087 raft_consensus.cc:482] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:57.346619 16087 raft_consensus.cc:3037] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:57.347658 16087 raft_consensus.cc:504] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.347960 16087 leader_election.cc:288] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 election: Requested vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:58:57.348241 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fcfc8dc4b90a4a6aa7e4035bbfdb4532" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:58:57.348290 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fcfc8dc4b90a4a6aa7e4035bbfdb4532" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:58:57.348345 15197 raft_consensus.cc:3037] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:57.348385 15061 raft_consensus.cc:3037] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:57.349051 15197 raft_consensus.cc:2454] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 1.
I20240502 01:58:57.349108 15061 raft_consensus.cc:2454] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 1.
I20240502 01:58:57.349434 15397 leader_election.cc:302] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5; no voters:
I20240502 01:58:57.349627 16087 raft_consensus.cc:2781] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:57.349720 16087 raft_consensus.cc:686] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Becoming Leader. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Running, Role: LEADER
I20240502 01:58:57.350064 16087 consensus_queue.cc:237] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
W20240502 01:58:57.350481 15517 tablet.cc:2369] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 01:58:57.352893 14913 catalog_manager.cc:5484] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } }
W20240502 01:58:57.488539 16109 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:57.488768 16109 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:57.491673 16109 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_0'
I20240502 01:58:57.591058 15197 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:57.592622 15061 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:57.593307 16100 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:57.593470 16089 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:57.594455 15195 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:57.594897 16087 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:57.598681 15061 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:57.600915 15466 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:57.601399 16088 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:57.601459 16087 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:57.602286 15331 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:57.604188 16088 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:57.605156 16137 mvcc.cc:205] Tried to move back new op lower bound from 7023063603579772928 to 7023063601940406272. Current Snapshot: MvccSnapshot[applied={T|T < 7023063603564544000}]
I20240502 01:58:57.608539 16140 mvcc.cc:205] Tried to move back new op lower bound from 7023063603579772928 to 7023063601940406272. Current Snapshot: MvccSnapshot[applied={T|T < 7023063603564544000}]
I20240502 01:58:57.614115 16139 mvcc.cc:205] Tried to move back new op lower bound from 7023063603559075840 to 7023063602191908864. Current Snapshot: MvccSnapshot[applied={T|T < 7023063603559075840}]
I20240502 01:58:57.622362 16136 mvcc.cc:205] Tried to move back new op lower bound from 7023063603559075840 to 7023063602191908864. Current Snapshot: MvccSnapshot[applied={T|T < 7023063603559075840}]
INSERT report
rows total: 2000
time total: 128.944 ms
time per row: 0.0644718 ms
I20240502 01:58:57.664506 14913 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54458:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:57.669677 15041 tablet_service.cc:1467] Processing CreateTablet for tablet fc07e430e3e74e3885f263881bd04976 (DEFAULT_TABLE table=rebalance_test_table_1 [id=1c35c23ce11f4c249032c42d4c499e42]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.669693 15446 tablet_service.cc:1467] Processing CreateTablet for tablet fc07e430e3e74e3885f263881bd04976 (DEFAULT_TABLE table=rebalance_test_table_1 [id=1c35c23ce11f4c249032c42d4c499e42]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.669780 15311 tablet_service.cc:1467] Processing CreateTablet for tablet fc07e430e3e74e3885f263881bd04976 (DEFAULT_TABLE table=rebalance_test_table_1 [id=1c35c23ce11f4c249032c42d4c499e42]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.669998 15311 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc07e430e3e74e3885f263881bd04976. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.670001 15446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc07e430e3e74e3885f263881bd04976. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.669998 15041 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc07e430e3e74e3885f263881bd04976. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.670914 15309 tablet_service.cc:1467] Processing CreateTablet for tablet 8ddc239a297a46e48e069b985cb7b350 (DEFAULT_TABLE table=rebalance_test_table_1 [id=1c35c23ce11f4c249032c42d4c499e42]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.670935 15176 tablet_service.cc:1467] Processing CreateTablet for tablet 8ddc239a297a46e48e069b985cb7b350 (DEFAULT_TABLE table=rebalance_test_table_1 [id=1c35c23ce11f4c249032c42d4c499e42]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.671165 15309 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8ddc239a297a46e48e069b985cb7b350. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.671192 15176 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8ddc239a297a46e48e069b985cb7b350. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.672463 15310 tablet_service.cc:1467] Processing CreateTablet for tablet 53edbd87d0d5471ca7626ca2071f75eb (DEFAULT_TABLE table=rebalance_test_table_1 [id=1c35c23ce11f4c249032c42d4c499e42]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.672665 15310 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 53edbd87d0d5471ca7626ca2071f75eb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.673342 16167 tablet_bootstrap.cc:492] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:58:57.673568 15041 tablet_service.cc:1467] Processing CreateTablet for tablet 53edbd87d0d5471ca7626ca2071f75eb (DEFAULT_TABLE table=rebalance_test_table_1 [id=1c35c23ce11f4c249032c42d4c499e42]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.673810 15041 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 53edbd87d0d5471ca7626ca2071f75eb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.673945 16165 tablet_bootstrap.cc:492] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:58:57.674311 16167 tablet_bootstrap.cc:654] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.675104 15446 tablet_service.cc:1467] Processing CreateTablet for tablet 53edbd87d0d5471ca7626ca2071f75eb (DEFAULT_TABLE table=rebalance_test_table_1 [id=1c35c23ce11f4c249032c42d4c499e42]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.675489 15446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 53edbd87d0d5471ca7626ca2071f75eb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.675542 16165 tablet_bootstrap.cc:654] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.675873 15040 tablet_service.cc:1467] Processing CreateTablet for tablet 8ddc239a297a46e48e069b985cb7b350 (DEFAULT_TABLE table=rebalance_test_table_1 [id=1c35c23ce11f4c249032c42d4c499e42]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:57.676112 15040 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8ddc239a297a46e48e069b985cb7b350. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:57.676537 16167 tablet_bootstrap.cc:492] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:58:57.676635 16167 ts_tablet_manager.cc:1367] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.001s
I20240502 01:58:57.677073 16167 raft_consensus.cc:348] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.677187 16167 raft_consensus.cc:374] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.677219 16167 raft_consensus.cc:729] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.677665 16165 tablet_bootstrap.cc:492] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:58:57.677696 16164 tablet_bootstrap.cc:492] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:58:57.677753 16165 ts_tablet_manager.cc:1367] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.002s
I20240502 01:58:57.678349 16165 raft_consensus.cc:348] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.678478 16165 raft_consensus.cc:374] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.678488 16164 tablet_bootstrap.cc:654] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.678514 16165 raft_consensus.cc:729] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.678650 16165 consensus_queue.cc:260] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.678946 16165 ts_tablet_manager.cc:1398] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:57.679118 16165 tablet_bootstrap.cc:492] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:58:57.679173 16166 tablet_bootstrap.cc:492] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:58:57.679865 16164 tablet_bootstrap.cc:492] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:58:57.679965 16165 tablet_bootstrap.cc:654] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.679957 16164 ts_tablet_manager.cc:1367] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:57.680418 16164 raft_consensus.cc:348] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.680514 16164 raft_consensus.cc:374] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.680544 16164 raft_consensus.cc:729] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.680662 16166 tablet_bootstrap.cc:654] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.680800 16167 consensus_queue.cc:260] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.680837 16164 consensus_queue.cc:260] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.681075 16164 ts_tablet_manager.cc:1398] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:57.681954 16164 tablet_bootstrap.cc:492] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:58:57.682089 16165 tablet_bootstrap.cc:492] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:58:57.682475 16165 ts_tablet_manager.cc:1367] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.001s
I20240502 01:58:57.683040 16164 tablet_bootstrap.cc:654] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.683225 16165 raft_consensus.cc:348] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.683326 16165 raft_consensus.cc:374] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.683362 16165 raft_consensus.cc:729] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.683602 16167 ts_tablet_manager.cc:1398] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.007s user 0.000s sys 0.001s
I20240502 01:58:57.683998 16165 consensus_queue.cc:260] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.684185 16164 tablet_bootstrap.cc:492] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:58:57.684249 16164 ts_tablet_manager.cc:1367] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:57.684295 16166 tablet_bootstrap.cc:492] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:58:57.684399 16167 tablet_bootstrap.cc:492] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:58:57.684413 16166 ts_tablet_manager.cc:1367] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 01:58:57.684698 16164 raft_consensus.cc:348] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.684794 16164 raft_consensus.cc:374] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.684828 16164 raft_consensus.cc:729] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.684836 16166 raft_consensus.cc:348] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.684937 16166 raft_consensus.cc:374] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.684928 16164 consensus_queue.cc:260] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.684973 16166 raft_consensus.cc:729] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.685151 16164 ts_tablet_manager.cc:1398] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:57.685127 16166 consensus_queue.cc:260] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.685329 16164 tablet_bootstrap.cc:492] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:58:57.685931 16167 tablet_bootstrap.cc:654] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.685985 16166 ts_tablet_manager.cc:1398] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:57.686163 16164 tablet_bootstrap.cc:654] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.686849 16165 ts_tablet_manager.cc:1398] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.004s user 0.000s sys 0.001s
I20240502 01:58:57.687008 16167 tablet_bootstrap.cc:492] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:58:57.687083 16167 ts_tablet_manager.cc:1367] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 01:58:57.687481 16164 tablet_bootstrap.cc:492] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:58:57.687443 16167 raft_consensus.cc:348] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.687536 16167 raft_consensus.cc:374] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.687572 16164 ts_tablet_manager.cc:1367] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:57.687566 16167 raft_consensus.cc:729] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.687691 16167 consensus_queue.cc:260] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.687950 16167 ts_tablet_manager.cc:1398] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:57.687979 16164 raft_consensus.cc:348] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.688081 16164 raft_consensus.cc:374] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.688112 16164 raft_consensus.cc:729] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.688115 16167 tablet_bootstrap.cc:492] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:58:57.688215 16164 consensus_queue.cc:260] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.688467 16164 ts_tablet_manager.cc:1398] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:57.688905 16167 tablet_bootstrap.cc:654] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:57.690316 16167 tablet_bootstrap.cc:492] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:58:57.690398 16167 ts_tablet_manager.cc:1367] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 01:58:57.690865 16167 raft_consensus.cc:348] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.691107 16167 raft_consensus.cc:374] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:57.691150 16167 raft_consensus.cc:729] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:58:57.691269 16167 consensus_queue.cc:260] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.691573 16167 ts_tablet_manager.cc:1398] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:57.721663 16093 raft_consensus.cc:482] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:57.721807 16093 raft_consensus.cc:504] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.722811 16093 leader_election.cc:288] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:58:57.726714 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8ddc239a297a46e48e069b985cb7b350" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" is_pre_election: true
I20240502 01:58:57.726711 15331 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8ddc239a297a46e48e069b985cb7b350" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:58:57.726861 15197 raft_consensus.cc:2454] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 0.
I20240502 01:58:57.726881 15331 raft_consensus.cc:2454] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 0.
I20240502 01:58:57.727231 14993 leader_election.cc:302] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:57.727501 16093 raft_consensus.cc:2781] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:57.727577 16093 raft_consensus.cc:482] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:57.727620 16093 raft_consensus.cc:3037] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:57.728459 16093 raft_consensus.cc:504] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.728781 16093 leader_election.cc:288] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 election: Requested vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:58:57.729108 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8ddc239a297a46e48e069b985cb7b350" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:58:57.729131 15331 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8ddc239a297a46e48e069b985cb7b350" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:58:57.729221 15331 raft_consensus.cc:3037] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:57.729235 15197 raft_consensus.cc:3037] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:57.729957 15331 raft_consensus.cc:2454] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 1.
I20240502 01:58:57.729966 15197 raft_consensus.cc:2454] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 1.
I20240502 01:58:57.730226 14993 leader_election.cc:302] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:57.730417 16093 raft_consensus.cc:2781] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:57.730654 16093 raft_consensus.cc:686] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Becoming Leader. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Running, Role: LEADER
I20240502 01:58:57.730963 16093 consensus_queue.cc:237] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:57.733217 14912 catalog_manager.cc:5484] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 reported cstate change: term changed from 0 to 1, leader changed from <none> to ddae2a636423489693c12b7d7d166c09 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:57.738718 16093 raft_consensus.cc:482] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:57.738801 16093 raft_consensus.cc:504] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.739161 16093 leader_election.cc:288] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:58:57.739465 15331 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "53edbd87d0d5471ca7626ca2071f75eb" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:58:57.739588 15331 raft_consensus.cc:2454] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 0.
I20240502 01:58:57.739836 14993 leader_election.cc:302] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:57.739997 16093 raft_consensus.cc:2781] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:57.740060 16093 raft_consensus.cc:482] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:57.740214 16093 raft_consensus.cc:3037] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:57.740921 16093 raft_consensus.cc:504] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.741194 16093 leader_election.cc:288] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 election: Requested vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:58:57.741492 15331 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "53edbd87d0d5471ca7626ca2071f75eb" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:58:57.741600 15331 raft_consensus.cc:3037] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:57.742506 15331 raft_consensus.cc:2454] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 1.
I20240502 01:58:57.742628 15466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "53edbd87d0d5471ca7626ca2071f75eb" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:58:57.742686 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "53edbd87d0d5471ca7626ca2071f75eb" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:58:57.742767 15466 raft_consensus.cc:2454] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 0.
I20240502 01:58:57.742789 14993 leader_election.cc:302] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:57.742954 16093 raft_consensus.cc:2781] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:57.743036 16093 raft_consensus.cc:686] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Becoming Leader. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Running, Role: LEADER
I20240502 01:58:57.743167 16093 consensus_queue.cc:237] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:58:57.744691 14912 catalog_manager.cc:5484] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 reported cstate change: term changed from 0 to 1, leader changed from <none> to ddae2a636423489693c12b7d7d166c09 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:58.040807 16091 raft_consensus.cc:482] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:58.040920 16091 raft_consensus.cc:504] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:58.041324 16091 leader_election.cc:288] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:58:58.041776 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc07e430e3e74e3885f263881bd04976" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09" is_pre_election: true
I20240502 01:58:58.041934 15061 raft_consensus.cc:2454] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 0.
I20240502 01:58:58.042270 15398 leader_election.cc:302] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, ddae2a636423489693c12b7d7d166c09; no voters:
I20240502 01:58:58.042484 16091 raft_consensus.cc:2781] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:58.042562 16091 raft_consensus.cc:482] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:58.042620 16091 raft_consensus.cc:3037] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:58.043517 16091 raft_consensus.cc:504] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:58.043888 16091 leader_election.cc:288] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 election: Requested vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:58:58.044204 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc07e430e3e74e3885f263881bd04976" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:58:58.044319 15061 raft_consensus.cc:3037] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:58.045323 15061 raft_consensus.cc:2454] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 1.
I20240502 01:58:58.045313 15331 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc07e430e3e74e3885f263881bd04976" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:58:58.045347 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc07e430e3e74e3885f263881bd04976" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:58:58.045437 15331 raft_consensus.cc:2454] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 0.
I20240502 01:58:58.045714 15398 leader_election.cc:302] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, ddae2a636423489693c12b7d7d166c09; no voters:
I20240502 01:58:58.045872 16091 raft_consensus.cc:2781] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:58.045958 16091 raft_consensus.cc:686] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Becoming Leader. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Running, Role: LEADER
I20240502 01:58:58.046104 16091 consensus_queue.cc:237] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:58.047628 14912 catalog_manager.cc:5484] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:58.231177 16184 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:58.231384 16184 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:58.234290 16184 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 01:58:58.235054 16093 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:58.238752 16093 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_1'
I20240502 01:58:58.272951 15465 raft_consensus.cc:3037] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:58.274751 16093 consensus_queue.cc:1035] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:58.290849 16179 consensus_queue.cc:1035] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:58.319994 15330 raft_consensus.cc:3037] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:58.320392 15061 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:58.320832 16091 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:58.321074 15330 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:58.327247 16091 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 112.924 ms
time per row: 0.0564618 ms
I20240502 01:58:58.386655 14912 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54458:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:58.391820 15446 tablet_service.cc:1467] Processing CreateTablet for tablet bf73e677bcd049869cadf9f8c9cd3434 (DEFAULT_TABLE table=rebalance_test_table_2 [id=73bf3d4779b34c7290b42f7f561e8e11]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.391921 15176 tablet_service.cc:1467] Processing CreateTablet for tablet bf73e677bcd049869cadf9f8c9cd3434 (DEFAULT_TABLE table=rebalance_test_table_2 [id=73bf3d4779b34c7290b42f7f561e8e11]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.392123 15446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bf73e677bcd049869cadf9f8c9cd3434. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.392230 15176 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bf73e677bcd049869cadf9f8c9cd3434. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.392135 15445 tablet_service.cc:1467] Processing CreateTablet for tablet 986e7c97e0734e958f364dc5d1167846 (DEFAULT_TABLE table=rebalance_test_table_2 [id=73bf3d4779b34c7290b42f7f561e8e11]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.392714 15311 tablet_service.cc:1467] Processing CreateTablet for tablet 986e7c97e0734e958f364dc5d1167846 (DEFAULT_TABLE table=rebalance_test_table_2 [id=73bf3d4779b34c7290b42f7f561e8e11]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.392865 15445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 986e7c97e0734e958f364dc5d1167846. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.392957 15311 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 986e7c97e0734e958f364dc5d1167846. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.394479 15174 tablet_service.cc:1467] Processing CreateTablet for tablet 986e7c97e0734e958f364dc5d1167846 (DEFAULT_TABLE table=rebalance_test_table_2 [id=73bf3d4779b34c7290b42f7f561e8e11]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.394748 15174 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 986e7c97e0734e958f364dc5d1167846. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.395108 15040 tablet_service.cc:1467] Processing CreateTablet for tablet bf73e677bcd049869cadf9f8c9cd3434 (DEFAULT_TABLE table=rebalance_test_table_2 [id=73bf3d4779b34c7290b42f7f561e8e11]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.395378 16247 tablet_bootstrap.cc:492] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:58:58.395398 15040 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bf73e677bcd049869cadf9f8c9cd3434. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.395505 16246 tablet_bootstrap.cc:492] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:58:58.396199 16247 tablet_bootstrap.cc:654] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:58.396484 16246 tablet_bootstrap.cc:654] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:58.396857 15175 tablet_service.cc:1467] Processing CreateTablet for tablet 3f938c5af51a43c9a23b753c4991e41d (DEFAULT_TABLE table=rebalance_test_table_2 [id=73bf3d4779b34c7290b42f7f561e8e11]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.397075 15175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3f938c5af51a43c9a23b753c4991e41d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.397470 16251 tablet_bootstrap.cc:492] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:58:58.397734 16248 tablet_bootstrap.cc:492] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:58:58.397984 16246 tablet_bootstrap.cc:492] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:58:58.398145 16246 ts_tablet_manager.cc:1367] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:58:58.398290 16251 tablet_bootstrap.cc:654] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:58.398797 16248 tablet_bootstrap.cc:654] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:58.398736 16246 raft_consensus.cc:348] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:58.398942 16246 raft_consensus.cc:374] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:58.399032 16246 raft_consensus.cc:729] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:58:58.399044 15041 tablet_service.cc:1467] Processing CreateTablet for tablet 3f938c5af51a43c9a23b753c4991e41d (DEFAULT_TABLE table=rebalance_test_table_2 [id=73bf3d4779b34c7290b42f7f561e8e11]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.399256 15041 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3f938c5af51a43c9a23b753c4991e41d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.399677 15445 tablet_service.cc:1467] Processing CreateTablet for tablet 3f938c5af51a43c9a23b753c4991e41d (DEFAULT_TABLE table=rebalance_test_table_2 [id=73bf3d4779b34c7290b42f7f561e8e11]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.399914 15445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3f938c5af51a43c9a23b753c4991e41d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.400143 16247 tablet_bootstrap.cc:492] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:58:58.399235 16246 consensus_queue.cc:260] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:58.400231 16247 ts_tablet_manager.cc:1367] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 01:58:58.400565 16247 raft_consensus.cc:348] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:58.400663 16247 raft_consensus.cc:374] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:58.400692 16247 raft_consensus.cc:729] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:58:58.400825 16247 consensus_queue.cc:260] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:58.401083 16247 ts_tablet_manager.cc:1398] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:58.401240 16247 tablet_bootstrap.cc:492] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:58:58.401497 16246 ts_tablet_manager.cc:1398] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:58.401641 16246 tablet_bootstrap.cc:492] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:58:58.402220 16247 tablet_bootstrap.cc:654] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:58.402446 16246 tablet_bootstrap.cc:654] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:58.403151 16247 tablet_bootstrap.cc:492] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:58:58.403219 16247 ts_tablet_manager.cc:1367] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:58.403620 16247 raft_consensus.cc:348] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:58.403718 16247 raft_consensus.cc:374] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:58.403767 16247 raft_consensus.cc:729] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:58:58.403872 16247 consensus_queue.cc:260] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:58.403914 16248 tablet_bootstrap.cc:492] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:58:58.404042 16246 tablet_bootstrap.cc:492] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:58:58.404053 16248 ts_tablet_manager.cc:1367] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 01:58:58.404130 16246 ts_tablet_manager.cc:1367] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:58.404125 16247 ts_tablet_manager.cc:1398] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:58.404287 16247 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:58:58.404585 16246 raft_consensus.cc:348] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:58.404697 16246 raft_consensus.cc:374] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:58.404739 16246 raft_consensus.cc:729] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:58:58.404665 16248 raft_consensus.cc:348] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:58.404871 16246 consensus_queue.cc:260] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:58.405012 16248 raft_consensus.cc:374] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:58.405098 16248 raft_consensus.cc:729] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:58:58.405169 16246 ts_tablet_manager.cc:1398] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:58.405194 16247 tablet_bootstrap.cc:654] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:58.405354 16246 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:58:58.405652 16248 consensus_queue.cc:260] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:58.406061 16251 tablet_bootstrap.cc:492] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:58:58.406144 16251 ts_tablet_manager.cc:1367] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.009s user 0.000s sys 0.002s
I20240502 01:58:58.406208 16246 tablet_bootstrap.cc:654] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:58.406579 16251 raft_consensus.cc:348] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:58.406697 16251 raft_consensus.cc:374] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:58.406733 16251 raft_consensus.cc:729] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:58:58.406852 16248 ts_tablet_manager.cc:1398] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:58.406857 16251 consensus_queue.cc:260] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:58.407089 16251 ts_tablet_manager.cc:1398] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:58.407224 16251 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:58:58.408066 16251 tablet_bootstrap.cc:654] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:58.409171 16251 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:58:58.409250 16251 ts_tablet_manager.cc:1367] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.001s
I20240502 01:58:58.409250 16246 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:58:58.409329 16246 ts_tablet_manager.cc:1367] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 01:58:58.409631 16251 raft_consensus.cc:348] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:58.409760 16251 raft_consensus.cc:374] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:58.409701 16246 raft_consensus.cc:348] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:58.409791 16251 raft_consensus.cc:729] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:58:58.409804 16246 raft_consensus.cc:374] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:58.409837 16246 raft_consensus.cc:729] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:58:58.409896 16251 consensus_queue.cc:260] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:58.409934 16246 consensus_queue.cc:260] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:58.410130 16251 ts_tablet_manager.cc:1398] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:58.410184 16247 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:58:58.410183 16246 ts_tablet_manager.cc:1398] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:58.410259 16247 ts_tablet_manager.cc:1367] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 01:58:58.410738 16247 raft_consensus.cc:348] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:58.410837 16247 raft_consensus.cc:374] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:58.410868 16247 raft_consensus.cc:729] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:58:58.411020 16247 consensus_queue.cc:260] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:58.411372 16247 ts_tablet_manager.cc:1398] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:58.478266 16087 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:58.478382 16087 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:58.478755 16087 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:58:58.479179 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" is_pre_election: true
I20240502 01:58:58.479179 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:58:58.479328 15330 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 0.
I20240502 01:58:58.479328 15197 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 0.
I20240502 01:58:58.479663 15397 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5; no voters:
I20240502 01:58:58.479823 16087 raft_consensus.cc:2781] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:58.479882 16087 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:58.479930 16087 raft_consensus.cc:3037] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:58.480690 16087 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:58.480984 16087 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 election: Requested vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:58:58.481304 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:58:58.481354 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:58:58.481415 15197 raft_consensus.cc:3037] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:58.481472 15330 raft_consensus.cc:3037] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:58.482335 15197 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 1.
I20240502 01:58:58.482335 15330 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 1.
I20240502 01:58:58.482666 15397 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5; no voters:
I20240502 01:58:58.482803 16087 raft_consensus.cc:2781] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:58.482908 16087 raft_consensus.cc:686] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Becoming Leader. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Running, Role: LEADER
I20240502 01:58:58.483070 16087 consensus_queue.cc:237] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:58.484750 14914 catalog_manager.cc:5484] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:58.573884 16087 raft_consensus.cc:482] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:58.574043 16087 raft_consensus.cc:504] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:58.574481 16087 leader_election.cc:288] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:58:58.574980 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3f938c5af51a43c9a23b753c4991e41d" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" is_pre_election: true
I20240502 01:58:58.574980 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3f938c5af51a43c9a23b753c4991e41d" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09" is_pre_election: true
I20240502 01:58:58.575129 15197 raft_consensus.cc:2454] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 0.
I20240502 01:58:58.575129 15061 raft_consensus.cc:2454] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 0.
I20240502 01:58:58.575477 15398 leader_election.cc:302] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, ddae2a636423489693c12b7d7d166c09; no voters:
I20240502 01:58:58.575666 16087 raft_consensus.cc:2781] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:58.575727 16087 raft_consensus.cc:482] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:58.575753 16087 raft_consensus.cc:3037] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:58.576555 16087 raft_consensus.cc:504] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:58.576884 16087 leader_election.cc:288] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 election: Requested vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:58:58.577179 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3f938c5af51a43c9a23b753c4991e41d" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:58:58.577179 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3f938c5af51a43c9a23b753c4991e41d" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:58:58.577282 15197 raft_consensus.cc:3037] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:58.577283 15061 raft_consensus.cc:3037] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:58.578150 15061 raft_consensus.cc:2454] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 1.
I20240502 01:58:58.578150 15197 raft_consensus.cc:2454] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 1.
I20240502 01:58:58.578433 15397 leader_election.cc:302] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5; no voters:
I20240502 01:58:58.578609 16087 raft_consensus.cc:2781] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:58.578722 16087 raft_consensus.cc:686] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Becoming Leader. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Running, Role: LEADER
I20240502 01:58:58.578881 16087 consensus_queue.cc:237] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:58.580502 14914 catalog_manager.cc:5484] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:58.740370 16087 raft_consensus.cc:482] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:58.740486 16087 raft_consensus.cc:504] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:58.740798 16087 leader_election.cc:288] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:58:58.741103 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bf73e677bcd049869cadf9f8c9cd3434" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" is_pre_election: true
I20240502 01:58:58.741127 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bf73e677bcd049869cadf9f8c9cd3434" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09" is_pre_election: true
I20240502 01:58:58.741251 15197 raft_consensus.cc:2454] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 0.
I20240502 01:58:58.741268 15061 raft_consensus.cc:2454] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 0.
I20240502 01:58:58.741503 15397 leader_election.cc:302] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5; no voters:
I20240502 01:58:58.741645 16087 raft_consensus.cc:2781] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:58.741699 16087 raft_consensus.cc:482] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:58.741724 16087 raft_consensus.cc:3037] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:58.742441 16087 raft_consensus.cc:504] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:58.742744 16087 leader_election.cc:288] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 election: Requested vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:58:58.743054 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bf73e677bcd049869cadf9f8c9cd3434" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:58:58.743093 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bf73e677bcd049869cadf9f8c9cd3434" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:58:58.743167 15197 raft_consensus.cc:3037] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:58.743222 15061 raft_consensus.cc:3037] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:58.743886 15197 raft_consensus.cc:2454] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 1.
I20240502 01:58:58.743889 15061 raft_consensus.cc:2454] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 1.
I20240502 01:58:58.744115 15397 leader_election.cc:302] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5; no voters:
I20240502 01:58:58.744280 16087 raft_consensus.cc:2781] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:58.744385 16087 raft_consensus.cc:686] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Becoming Leader. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Running, Role: LEADER
I20240502 01:58:58.744488 16087 consensus_queue.cc:237] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:58.746129 14914 catalog_manager.cc:5484] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:58.859372 16263 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:58.859570 16263 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:58.862408 16263 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 01:58:58.878546 16087 consensus_queue.cc:1035] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_2'
I20240502 01:58:58.881516 16091 consensus_queue.cc:1035] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:58.934137 15060 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:58.934515 15061 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:58.934939 16087 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:58.935701 15195 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:58.936844 15197 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:58.934584 16298 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.002s
I20240502 01:58:58.938148 16091 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:58.938287 16087 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 82.417 ms
time per row: 0.0412085 ms
I20240502 01:58:58.985570 14915 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54458:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:58.990109 15041 tablet_service.cc:1467] Processing CreateTablet for tablet 6db29f2516e1486782be0312c9bcac8b (DEFAULT_TABLE table=rebalance_test_table_3 [id=0b29d58fe9c74d1cb4345bb6cfbd0535]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.990178 15175 tablet_service.cc:1467] Processing CreateTablet for tablet 6db29f2516e1486782be0312c9bcac8b (DEFAULT_TABLE table=rebalance_test_table_3 [id=0b29d58fe9c74d1cb4345bb6cfbd0535]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.990406 15041 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6db29f2516e1486782be0312c9bcac8b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.990437 15175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6db29f2516e1486782be0312c9bcac8b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.990579 15445 tablet_service.cc:1467] Processing CreateTablet for tablet 091d27d51acc4db0a1c7d20deeff5c10 (DEFAULT_TABLE table=rebalance_test_table_3 [id=0b29d58fe9c74d1cb4345bb6cfbd0535]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.990834 15445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 091d27d51acc4db0a1c7d20deeff5c10. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.991448 15311 tablet_service.cc:1467] Processing CreateTablet for tablet 6db29f2516e1486782be0312c9bcac8b (DEFAULT_TABLE table=rebalance_test_table_3 [id=0b29d58fe9c74d1cb4345bb6cfbd0535]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.991716 15311 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6db29f2516e1486782be0312c9bcac8b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.992676 16324 tablet_bootstrap.cc:492] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:58:58.992769 16326 tablet_bootstrap.cc:492] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:58:58.993119 15041 tablet_service.cc:1467] Processing CreateTablet for tablet 091d27d51acc4db0a1c7d20deeff5c10 (DEFAULT_TABLE table=rebalance_test_table_3 [id=0b29d58fe9c74d1cb4345bb6cfbd0535]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.993324 15041 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 091d27d51acc4db0a1c7d20deeff5c10. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.993520 16324 tablet_bootstrap.cc:654] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:58.993544 16326 tablet_bootstrap.cc:654] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:58.993893 15310 tablet_service.cc:1467] Processing CreateTablet for tablet 091d27d51acc4db0a1c7d20deeff5c10 (DEFAULT_TABLE table=rebalance_test_table_3 [id=0b29d58fe9c74d1cb4345bb6cfbd0535]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.994120 15310 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 091d27d51acc4db0a1c7d20deeff5c10. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.994369 15309 tablet_service.cc:1467] Processing CreateTablet for tablet 5804597cc507483dba622e6ec44e5ead (DEFAULT_TABLE table=rebalance_test_table_3 [id=0b29d58fe9c74d1cb4345bb6cfbd0535]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.994511 16326 tablet_bootstrap.cc:492] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:58:58.994535 15309 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5804597cc507483dba622e6ec44e5ead. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.994575 16326 ts_tablet_manager.cc:1367] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:58.995020 16326 raft_consensus.cc:348] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:58.995117 16326 raft_consensus.cc:374] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:58.995146 16326 raft_consensus.cc:729] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:58:58.995157 16325 tablet_bootstrap.cc:492] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:58:58.995254 16326 consensus_queue.cc:260] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:58.995509 16326 ts_tablet_manager.cc:1398] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:58.995923 16325 tablet_bootstrap.cc:654] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:58.996766 15175 tablet_service.cc:1467] Processing CreateTablet for tablet 5804597cc507483dba622e6ec44e5ead (DEFAULT_TABLE table=rebalance_test_table_3 [id=0b29d58fe9c74d1cb4345bb6cfbd0535]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.996971 15175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5804597cc507483dba622e6ec44e5ead. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.997144 16325 tablet_bootstrap.cc:492] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:58:58.997225 16325 ts_tablet_manager.cc:1367] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:58.997788 16325 raft_consensus.cc:348] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:58.997874 16327 tablet_bootstrap.cc:492] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:58:58.997889 16325 raft_consensus.cc:374] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:58.997920 16325 raft_consensus.cc:729] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:58:58.998054 16325 consensus_queue.cc:260] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:58.998706 16327 tablet_bootstrap.cc:654] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:58.998904 15445 tablet_service.cc:1467] Processing CreateTablet for tablet 5804597cc507483dba622e6ec44e5ead (DEFAULT_TABLE table=rebalance_test_table_3 [id=0b29d58fe9c74d1cb4345bb6cfbd0535]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:58.999122 15445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5804597cc507483dba622e6ec44e5ead. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:58.999713 16324 tablet_bootstrap.cc:492] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:58:58.999797 16324 ts_tablet_manager.cc:1367] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 01:58:59.000041 16325 ts_tablet_manager.cc:1398] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:59.000216 16324 raft_consensus.cc:348] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.000312 16324 raft_consensus.cc:374] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.000340 16324 raft_consensus.cc:729] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.000445 16324 consensus_queue.cc:260] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.000684 16325 tablet_bootstrap.cc:492] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:58:59.000710 16324 ts_tablet_manager.cc:1398] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:59.000838 16324 tablet_bootstrap.cc:492] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:58:59.000860 16326 tablet_bootstrap.cc:492] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:58:59.001489 16325 tablet_bootstrap.cc:654] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:59.001639 16326 tablet_bootstrap.cc:654] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:59.001847 16324 tablet_bootstrap.cc:654] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:59.002475 16325 tablet_bootstrap.cc:492] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:58:59.002545 16325 ts_tablet_manager.cc:1367] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:59.002864 16324 tablet_bootstrap.cc:492] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:58:59.002936 16324 ts_tablet_manager.cc:1367] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:59.003010 16325 raft_consensus.cc:348] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.003111 16325 raft_consensus.cc:374] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.003140 16325 raft_consensus.cc:729] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.003242 16325 consensus_queue.cc:260] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.003340 16324 raft_consensus.cc:348] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.003418 16324 raft_consensus.cc:374] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.003451 16324 raft_consensus.cc:729] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.003504 16325 ts_tablet_manager.cc:1398] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:59.003559 16324 consensus_queue.cc:260] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.003854 16324 ts_tablet_manager.cc:1398] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:59.004230 16326 tablet_bootstrap.cc:492] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:58:59.004344 16326 ts_tablet_manager.cc:1367] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:58:59.004390 16327 tablet_bootstrap.cc:492] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:58:59.004478 16327 ts_tablet_manager.cc:1367] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.001s
I20240502 01:58:59.004801 16326 raft_consensus.cc:348] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.004889 16326 raft_consensus.cc:374] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.004920 16326 raft_consensus.cc:729] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.004916 16327 raft_consensus.cc:348] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.005040 16327 raft_consensus.cc:374] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.005074 16327 raft_consensus.cc:729] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.005033 16326 consensus_queue.cc:260] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.005180 16327 consensus_queue.cc:260] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.005306 16326 ts_tablet_manager.cc:1398] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:59.005461 16327 ts_tablet_manager.cc:1398] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:59.005640 16327 tablet_bootstrap.cc:492] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:58:59.006498 16327 tablet_bootstrap.cc:654] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:59.007766 16327 tablet_bootstrap.cc:492] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:58:59.007838 16327 ts_tablet_manager.cc:1367] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:59.008260 16327 raft_consensus.cc:348] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.008373 16327 raft_consensus.cc:374] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.008409 16327 raft_consensus.cc:729] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.008529 16327 consensus_queue.cc:260] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.008800 16327 ts_tablet_manager.cc:1398] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:59.008950 16327 tablet_bootstrap.cc:492] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:58:59.009711 16327 tablet_bootstrap.cc:654] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:59.010722 16327 tablet_bootstrap.cc:492] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:58:59.010802 16327 ts_tablet_manager.cc:1367] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.001s
I20240502 01:58:59.011111 16327 raft_consensus.cc:348] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.011185 16327 raft_consensus.cc:374] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.011216 16327 raft_consensus.cc:729] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.011325 16327 consensus_queue.cc:260] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.011544 16327 ts_tablet_manager.cc:1398] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:59.092104 16315 raft_consensus.cc:482] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:59.092233 16315 raft_consensus.cc:504] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.092692 16315 leader_election.cc:288] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:58:59.093078 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5804597cc507483dba622e6ec44e5ead" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:58:59.093143 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5804597cc507483dba622e6ec44e5ead" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:58:59.093230 15465 raft_consensus.cc:2454] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:58:59.093271 15330 raft_consensus.cc:2454] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:58:59.093519 15129 leader_election.cc:302] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:59.093683 16315 raft_consensus.cc:2781] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:59.093750 16315 raft_consensus.cc:482] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:59.093781 16315 raft_consensus.cc:3037] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.094794 16315 raft_consensus.cc:504] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.095099 16315 leader_election.cc:288] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Requested vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:58:59.095409 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5804597cc507483dba622e6ec44e5ead" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:58:59.095449 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5804597cc507483dba622e6ec44e5ead" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:58:59.095539 15465 raft_consensus.cc:3037] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.095561 15330 raft_consensus.cc:3037] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.096488 15330 raft_consensus.cc:2454] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:58:59.096588 15465 raft_consensus.cc:2454] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:58:59.096773 15129 leader_election.cc:302] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:59.096925 16315 raft_consensus.cc:2781] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:59.097005 16315 raft_consensus.cc:686] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Becoming Leader. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Running, Role: LEADER
I20240502 01:58:59.097131 16315 consensus_queue.cc:237] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.098809 14913 catalog_manager.cc:5484] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: term changed from 0 to 1, leader changed from <none> to 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66). New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:59.099341 16216 raft_consensus.cc:482] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:59.099436 16216 raft_consensus.cc:504] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.099871 16216 leader_election.cc:288] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:58:59.100662 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "091d27d51acc4db0a1c7d20deeff5c10" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09" is_pre_election: true
I20240502 01:58:59.100791 15061 raft_consensus.cc:2454] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 0.
I20240502 01:58:59.101140 15265 leader_election.cc:302] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:59.101298 16216 raft_consensus.cc:2781] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:59.101358 16216 raft_consensus.cc:482] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:59.101433 16216 raft_consensus.cc:3037] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.102257 16216 raft_consensus.cc:504] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.102564 16216 leader_election.cc:288] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 election: Requested vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:58:59.102968 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "091d27d51acc4db0a1c7d20deeff5c10" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:58:59.103085 15061 raft_consensus.cc:3037] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.103955 15061 raft_consensus.cc:2454] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 1.
I20240502 01:58:59.104238 15265 leader_election.cc:302] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:59.104424 16216 raft_consensus.cc:2781] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:59.104606 16216 raft_consensus.cc:686] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Becoming Leader. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Running, Role: LEADER
I20240502 01:58:59.104764 16216 consensus_queue.cc:237] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.106458 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "091d27d51acc4db0a1c7d20deeff5c10" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:58:59.106518 15466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "091d27d51acc4db0a1c7d20deeff5c10" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:58:59.106631 15465 raft_consensus.cc:2454] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 0.
I20240502 01:58:59.107182 14913 catalog_manager.cc:5484] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: term changed from 0 to 1, leader changed from <none> to f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67). New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } }
I20240502 01:58:59.139564 16216 raft_consensus.cc:482] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:59.139690 16216 raft_consensus.cc:504] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.140101 16216 leader_election.cc:288] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:58:59.140501 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6db29f2516e1486782be0312c9bcac8b" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09" is_pre_election: true
I20240502 01:58:59.140507 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6db29f2516e1486782be0312c9bcac8b" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" is_pre_election: true
I20240502 01:58:59.140647 15061 raft_consensus.cc:2454] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 0.
I20240502 01:58:59.140647 15197 raft_consensus.cc:2454] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 0.
I20240502 01:58:59.140951 15264 leader_election.cc:302] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:59.141130 16216 raft_consensus.cc:2781] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:59.141194 16216 raft_consensus.cc:482] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:59.141222 16216 raft_consensus.cc:3037] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.142012 16216 raft_consensus.cc:504] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.142313 16216 leader_election.cc:288] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 election: Requested vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:58:59.142638 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6db29f2516e1486782be0312c9bcac8b" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:58:59.142668 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6db29f2516e1486782be0312c9bcac8b" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:58:59.142750 15197 raft_consensus.cc:3037] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.142773 15061 raft_consensus.cc:3037] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.143463 15197 raft_consensus.cc:2454] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 1.
I20240502 01:58:59.143507 15061 raft_consensus.cc:2454] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 1.
I20240502 01:58:59.143757 15265 leader_election.cc:302] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:59.143961 16216 raft_consensus.cc:2781] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:59.144037 16216 raft_consensus.cc:686] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Becoming Leader. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Running, Role: LEADER
I20240502 01:58:59.144198 16216 consensus_queue.cc:237] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:58:59.145835 14913 catalog_manager.cc:5484] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: term changed from 0 to 1, leader changed from <none> to f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67). New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } }
W20240502 01:58:59.287267 16344 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:59.287508 16344 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:59.290210 16344 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_3'
I20240502 01:58:59.366425 15061 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:59.366413 15465 raft_consensus.cc:3037] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.366887 16340 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:59.367475 15465 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:59.368645 15330 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:59.368997 16315 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:59.371193 16216 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:59.371870 15465 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:59.372268 16315 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:59.372365 15197 raft_consensus.cc:1261] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:59.373123 15060 raft_consensus.cc:1261] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:59.373509 16381 consensus_queue.cc:1035] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:58:59.375342 16381 consensus_queue.cc:1035] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 93.8862 ms
time per row: 0.0469431 ms
I20240502 01:58:59.430554 14906 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54458:
name: "rebalance_test_table_4"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:58:59.435989 15175 tablet_service.cc:1467] Processing CreateTablet for tablet 1eb15147ff6c4d0b83b12dec736fc576 (DEFAULT_TABLE table=rebalance_test_table_4 [id=0fd3b28283df457db390a2882ebcfef5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:59.436305 15175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1eb15147ff6c4d0b83b12dec736fc576. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:59.436383 15174 tablet_service.cc:1467] Processing CreateTablet for tablet f97c5c9bb4c84957857efc76d1c7717f (DEFAULT_TABLE table=rebalance_test_table_4 [id=0fd3b28283df457db390a2882ebcfef5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:59.436621 15174 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f97c5c9bb4c84957857efc76d1c7717f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:59.438164 16325 tablet_bootstrap.cc:492] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:58:59.438376 15309 tablet_service.cc:1467] Processing CreateTablet for tablet 1eb15147ff6c4d0b83b12dec736fc576 (DEFAULT_TABLE table=rebalance_test_table_4 [id=0fd3b28283df457db390a2882ebcfef5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:59.438637 15309 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1eb15147ff6c4d0b83b12dec736fc576. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:59.438567 15039 tablet_service.cc:1467] Processing CreateTablet for tablet ce85785e670a4264b44797bb27cb4ecb (DEFAULT_TABLE table=rebalance_test_table_4 [id=0fd3b28283df457db390a2882ebcfef5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:59.438865 15039 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ce85785e670a4264b44797bb27cb4ecb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:59.439172 16325 tablet_bootstrap.cc:654] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:59.440424 16327 tablet_bootstrap.cc:492] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:58:59.440654 15041 tablet_service.cc:1467] Processing CreateTablet for tablet 1eb15147ff6c4d0b83b12dec736fc576 (DEFAULT_TABLE table=rebalance_test_table_4 [id=0fd3b28283df457db390a2882ebcfef5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:59.440876 15041 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1eb15147ff6c4d0b83b12dec736fc576. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:59.441207 16327 tablet_bootstrap.cc:654] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:59.441220 16324 tablet_bootstrap.cc:492] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:58:59.441958 16324 tablet_bootstrap.cc:654] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:59.442564 16327 tablet_bootstrap.cc:492] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:58:59.442752 16327 ts_tablet_manager.cc:1367] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 01:58:59.443017 15040 tablet_service.cc:1467] Processing CreateTablet for tablet f97c5c9bb4c84957857efc76d1c7717f (DEFAULT_TABLE table=rebalance_test_table_4 [id=0fd3b28283df457db390a2882ebcfef5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:59.443249 15040 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f97c5c9bb4c84957857efc76d1c7717f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:59.443176 16327 raft_consensus.cc:348] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.443292 16327 raft_consensus.cc:374] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.443331 16327 raft_consensus.cc:729] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.443456 16327 consensus_queue.cc:260] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.443732 16327 ts_tablet_manager.cc:1398] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:59.445595 15309 tablet_service.cc:1467] Processing CreateTablet for tablet ce85785e670a4264b44797bb27cb4ecb (DEFAULT_TABLE table=rebalance_test_table_4 [id=0fd3b28283df457db390a2882ebcfef5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:59.445829 15309 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ce85785e670a4264b44797bb27cb4ecb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:59.446261 15445 tablet_service.cc:1467] Processing CreateTablet for tablet f97c5c9bb4c84957857efc76d1c7717f (DEFAULT_TABLE table=rebalance_test_table_4 [id=0fd3b28283df457db390a2882ebcfef5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:59.446261 15446 tablet_service.cc:1467] Processing CreateTablet for tablet ce85785e670a4264b44797bb27cb4ecb (DEFAULT_TABLE table=rebalance_test_table_4 [id=0fd3b28283df457db390a2882ebcfef5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:58:59.446867 15445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f97c5c9bb4c84957857efc76d1c7717f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:59.447005 15446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ce85785e670a4264b44797bb27cb4ecb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:58:59.447729 16327 tablet_bootstrap.cc:492] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:58:59.448559 16327 tablet_bootstrap.cc:654] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:59.449187 16326 tablet_bootstrap.cc:492] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:58:59.449664 16327 tablet_bootstrap.cc:492] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:58:59.449743 16327 ts_tablet_manager.cc:1367] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:58:59.450057 16326 tablet_bootstrap.cc:654] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:59.450196 16327 raft_consensus.cc:348] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.450350 16327 raft_consensus.cc:374] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.450397 16327 raft_consensus.cc:729] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.450526 16327 consensus_queue.cc:260] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.450832 16327 ts_tablet_manager.cc:1398] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:59.452451 16326 tablet_bootstrap.cc:492] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:58:59.452543 16326 ts_tablet_manager.cc:1367] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:58:59.452634 16324 tablet_bootstrap.cc:492] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:58:59.452729 16324 ts_tablet_manager.cc:1367] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.012s user 0.001s sys 0.000s
I20240502 01:58:59.452952 16326 raft_consensus.cc:348] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.453047 16326 raft_consensus.cc:374] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.453076 16326 raft_consensus.cc:729] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.453182 16326 consensus_queue.cc:260] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.453202 16324 raft_consensus.cc:348] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.453315 16324 raft_consensus.cc:374] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.453352 16324 raft_consensus.cc:729] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.453460 16326 ts_tablet_manager.cc:1398] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:59.453473 16324 consensus_queue.cc:260] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.453630 16326 tablet_bootstrap.cc:492] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:58:59.453732 16324 ts_tablet_manager.cc:1398] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:59.453894 16324 tablet_bootstrap.cc:492] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:58:59.454591 16326 tablet_bootstrap.cc:654] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:59.454687 16324 tablet_bootstrap.cc:654] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:59.455727 16324 tablet_bootstrap.cc:492] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:58:59.455816 16324 ts_tablet_manager.cc:1367] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 01:58:59.456243 16324 raft_consensus.cc:348] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.456341 16324 raft_consensus.cc:374] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.456382 16324 raft_consensus.cc:729] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.456516 16326 tablet_bootstrap.cc:492] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:58:59.456513 16324 consensus_queue.cc:260] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.456590 16326 ts_tablet_manager.cc:1367] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.001s
I20240502 01:58:59.456866 16324 ts_tablet_manager.cc:1398] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:59.457008 16324 tablet_bootstrap.cc:492] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:58:59.457006 16326 raft_consensus.cc:348] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:59.457103 16326 raft_consensus.cc:374] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.457130 16326 raft_consensus.cc:729] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.457245 16326 consensus_queue.cc:260] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:59.457571 16326 ts_tablet_manager.cc:1398] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:58:59.457809 16324 tablet_bootstrap.cc:654] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:59.458245 16325 tablet_bootstrap.cc:492] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:58:59.458335 16325 ts_tablet_manager.cc:1367] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.020s user 0.001s sys 0.000s
I20240502 01:58:59.458819 16324 tablet_bootstrap.cc:492] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:58:59.458788 16325 raft_consensus.cc:348] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.458870 16325 raft_consensus.cc:374] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.458902 16325 raft_consensus.cc:729] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.458904 16324 ts_tablet_manager.cc:1367] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 01:58:59.459028 16325 consensus_queue.cc:260] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.459249 16325 ts_tablet_manager.cc:1398] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:59.459301 16324 raft_consensus.cc:348] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:59.459386 16324 raft_consensus.cc:374] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.459414 16324 raft_consensus.cc:729] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.459424 16325 tablet_bootstrap.cc:492] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:58:59.459537 16324 consensus_queue.cc:260] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:59.459785 16324 ts_tablet_manager.cc:1398] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:58:59.460315 16325 tablet_bootstrap.cc:654] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:58:59.461576 16325 tablet_bootstrap.cc:492] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:58:59.461676 16325 ts_tablet_manager.cc:1367] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:58:59.462060 16325 raft_consensus.cc:348] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:59.462144 16325 raft_consensus.cc:374] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:58:59.462164 16325 raft_consensus.cc:729] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:58:59.462250 16325 consensus_queue.cc:260] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:59.462460 16325 ts_tablet_manager.cc:1398] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:58:59.616319 16088 raft_consensus.cc:482] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:59.616454 16088 raft_consensus.cc:504] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:59.616953 16088 leader_election.cc:288] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:58:59.617727 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f97c5c9bb4c84957857efc76d1c7717f" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:58:59.617866 15465 raft_consensus.cc:2454] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:58:59.618330 15130 leader_election.cc:302] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5; no voters:
I20240502 01:58:59.618499 16088 raft_consensus.cc:2781] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:59.618561 16088 raft_consensus.cc:482] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:59.618624 16088 raft_consensus.cc:3037] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.619545 16088 raft_consensus.cc:504] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:59.619839 16088 leader_election.cc:288] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Requested vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:58:59.620215 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f97c5c9bb4c84957857efc76d1c7717f" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:58:59.620322 15465 raft_consensus.cc:3037] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.621188 15465 raft_consensus.cc:2454] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:58:59.621455 15130 leader_election.cc:302] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5; no voters:
I20240502 01:58:59.621495 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f97c5c9bb4c84957857efc76d1c7717f" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:58:59.621626 15061 raft_consensus.cc:3037] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.622385 15061 raft_consensus.cc:2454] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:58:59.622648 15060 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f97c5c9bb4c84957857efc76d1c7717f" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09" is_pre_election: true
I20240502 01:58:59.622781 15060 raft_consensus.cc:2362] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1. Re-sending same reply.
I20240502 01:58:59.622937 16088 raft_consensus.cc:2781] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:59.623023 16088 raft_consensus.cc:686] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Becoming Leader. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Running, Role: LEADER
I20240502 01:58:59.623150 16088 consensus_queue.cc:237] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:58:59.624827 14908 catalog_manager.cc:5484] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: term changed from 0 to 1, leader changed from <none> to 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66). New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } }
I20240502 01:58:59.662058 16387 raft_consensus.cc:482] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:59.662202 16387 raft_consensus.cc:504] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.662657 16387 leader_election.cc:288] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:58:59.663127 15195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1eb15147ff6c4d0b83b12dec736fc576" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" is_pre_election: true
I20240502 01:58:59.663153 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1eb15147ff6c4d0b83b12dec736fc576" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:58:59.663298 15195 raft_consensus.cc:2454] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 0.
I20240502 01:58:59.663316 15330 raft_consensus.cc:2454] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 0.
I20240502 01:58:59.663637 14993 leader_election.cc:302] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:59.663810 16387 raft_consensus.cc:2781] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:59.663870 16387 raft_consensus.cc:482] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:59.663908 16387 raft_consensus.cc:3037] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.664767 16387 raft_consensus.cc:504] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.665081 16387 leader_election.cc:288] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 election: Requested vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:58:59.665452 15195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1eb15147ff6c4d0b83b12dec736fc576" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:58:59.665468 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1eb15147ff6c4d0b83b12dec736fc576" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:58:59.665556 15195 raft_consensus.cc:3037] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.665578 15330 raft_consensus.cc:3037] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.666349 15330 raft_consensus.cc:2454] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 1.
I20240502 01:58:59.666363 15195 raft_consensus.cc:2454] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 1.
I20240502 01:58:59.666687 14993 leader_election.cc:302] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, ddae2a636423489693c12b7d7d166c09; no voters:
I20240502 01:58:59.666839 16387 raft_consensus.cc:2781] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:59.666927 16387 raft_consensus.cc:686] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Becoming Leader. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Running, Role: LEADER
I20240502 01:58:59.667057 16387 consensus_queue.cc:237] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.668820 14908 catalog_manager.cc:5484] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 reported cstate change: term changed from 0 to 1, leader changed from <none> to ddae2a636423489693c12b7d7d166c09 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } }
I20240502 01:58:59.779560 16386 raft_consensus.cc:482] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:58:59.779685 16386 raft_consensus.cc:504] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.780110 16386 leader_election.cc:288] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:58:59.780500 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ce85785e670a4264b44797bb27cb4ecb" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:58:59.780524 15060 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ce85785e670a4264b44797bb27cb4ecb" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09" is_pre_election: true
I20240502 01:58:59.780634 15465 raft_consensus.cc:2454] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 0.
I20240502 01:58:59.780668 15060 raft_consensus.cc:2454] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 0.
I20240502 01:58:59.780900 15265 leader_election.cc:302] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:59.781059 16386 raft_consensus.cc:2781] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:58:59.781114 16386 raft_consensus.cc:482] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:58:59.781157 16386 raft_consensus.cc:3037] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.781910 16386 raft_consensus.cc:504] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.782181 16386 leader_election.cc:288] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 election: Requested vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:58:59.782500 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ce85785e670a4264b44797bb27cb4ecb" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:58:59.782541 15060 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ce85785e670a4264b44797bb27cb4ecb" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:58:59.782642 15465 raft_consensus.cc:3037] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.782680 15060 raft_consensus.cc:3037] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:58:59.783352 15465 raft_consensus.cc:2454] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 1.
I20240502 01:58:59.783433 15060 raft_consensus.cc:2454] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 1.
I20240502 01:58:59.783624 15265 leader_election.cc:302] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:58:59.783787 16386 raft_consensus.cc:2781] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:58:59.783870 16386 raft_consensus.cc:686] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Becoming Leader. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Running, Role: LEADER
I20240502 01:58:59.784005 16386 consensus_queue.cc:237] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:58:59.785599 14908 catalog_manager.cc:5484] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: term changed from 0 to 1, leader changed from <none> to f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67). New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } }
W20240502 01:58:59.913944 16404 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:58:59.914152 16404 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:58:59.917095 16404 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_4'
I20240502 01:58:59.999083 15465 raft_consensus.cc:1261] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:58:59.999686 16314 consensus_queue.cc:1035] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:00.000046 15330 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:00.000278 15195 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:00.000720 16387 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:00.002300 16387 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:00.003258 15060 raft_consensus.cc:1261] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:00.005574 16088 consensus_queue.cc:1035] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:00.008423 15061 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:00.009301 16386 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:00.010879 15466 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:00.011277 16386 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 113.645 ms
time per row: 0.0568224 ms
I20240502 01:59:00.068181 14906 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54458:
name: "rebalance_test_table_5"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:59:00.073067 15445 tablet_service.cc:1467] Processing CreateTablet for tablet 490e37fac0844a08a07a4b281bc33bf9 (DEFAULT_TABLE table=rebalance_test_table_5 [id=3bc69a0a1e474b9fac7330d68770bfbe]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.073164 15309 tablet_service.cc:1467] Processing CreateTablet for tablet 490e37fac0844a08a07a4b281bc33bf9 (DEFAULT_TABLE table=rebalance_test_table_5 [id=3bc69a0a1e474b9fac7330d68770bfbe]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.073149 15446 tablet_service.cc:1467] Processing CreateTablet for tablet ac5ec2c673cb4b988d1099e0442ec63f (DEFAULT_TABLE table=rebalance_test_table_5 [id=3bc69a0a1e474b9fac7330d68770bfbe]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.073348 15445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 490e37fac0844a08a07a4b281bc33bf9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.073379 15309 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 490e37fac0844a08a07a4b281bc33bf9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.073753 15174 tablet_service.cc:1467] Processing CreateTablet for tablet 490e37fac0844a08a07a4b281bc33bf9 (DEFAULT_TABLE table=rebalance_test_table_5 [id=3bc69a0a1e474b9fac7330d68770bfbe]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.074002 15174 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 490e37fac0844a08a07a4b281bc33bf9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.074012 15040 tablet_service.cc:1467] Processing CreateTablet for tablet d1c832c721fb4b15bc6d266a1c295401 (DEFAULT_TABLE table=rebalance_test_table_5 [id=3bc69a0a1e474b9fac7330d68770bfbe]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.074227 15040 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d1c832c721fb4b15bc6d266a1c295401. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.076253 16468 tablet_bootstrap.cc:492] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:59:00.076370 15446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ac5ec2c673cb4b988d1099e0442ec63f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.076815 15310 tablet_service.cc:1467] Processing CreateTablet for tablet d1c832c721fb4b15bc6d266a1c295401 (DEFAULT_TABLE table=rebalance_test_table_5 [id=3bc69a0a1e474b9fac7330d68770bfbe]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.076830 15309 tablet_service.cc:1467] Processing CreateTablet for tablet ac5ec2c673cb4b988d1099e0442ec63f (DEFAULT_TABLE table=rebalance_test_table_5 [id=3bc69a0a1e474b9fac7330d68770bfbe]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.077107 15310 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d1c832c721fb4b15bc6d266a1c295401. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.077515 15309 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ac5ec2c673cb4b988d1099e0442ec63f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.077607 16468 tablet_bootstrap.cc:654] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.073753 15175 tablet_service.cc:1467] Processing CreateTablet for tablet ac5ec2c673cb4b988d1099e0442ec63f (DEFAULT_TABLE table=rebalance_test_table_5 [id=3bc69a0a1e474b9fac7330d68770bfbe]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.078418 15175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ac5ec2c673cb4b988d1099e0442ec63f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.079535 15174 tablet_service.cc:1467] Processing CreateTablet for tablet d1c832c721fb4b15bc6d266a1c295401 (DEFAULT_TABLE table=rebalance_test_table_5 [id=3bc69a0a1e474b9fac7330d68770bfbe]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.079725 15174 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d1c832c721fb4b15bc6d266a1c295401. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.080281 16466 tablet_bootstrap.cc:492] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:59:00.081158 16466 tablet_bootstrap.cc:654] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.081486 16469 tablet_bootstrap.cc:492] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:59:00.081812 16467 tablet_bootstrap.cc:492] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:59:00.082177 16466 tablet_bootstrap.cc:492] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:59:00.082260 16466 ts_tablet_manager.cc:1367] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:00.082325 16469 tablet_bootstrap.cc:654] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.082690 16467 tablet_bootstrap.cc:654] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.082690 16466 raft_consensus.cc:348] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:00.082805 16466 raft_consensus.cc:374] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.082841 16466 raft_consensus.cc:729] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.082971 16466 consensus_queue.cc:260] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:00.083403 16469 tablet_bootstrap.cc:492] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:59:00.083494 16469 ts_tablet_manager.cc:1367] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:59:00.083878 16466 ts_tablet_manager.cc:1398] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:00.083945 16469 raft_consensus.cc:348] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.084050 16469 raft_consensus.cc:374] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.084060 16466 tablet_bootstrap.cc:492] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:59:00.084080 16469 raft_consensus.cc:729] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.084187 16469 consensus_queue.cc:260] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.084481 16469 ts_tablet_manager.cc:1398] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:00.084816 16468 tablet_bootstrap.cc:492] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:59:00.084897 16466 tablet_bootstrap.cc:654] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.084904 16468 ts_tablet_manager.cc:1367] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.009s user 0.002s sys 0.000s
I20240502 01:59:00.085600 16467 tablet_bootstrap.cc:492] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:59:00.085697 16467 ts_tablet_manager.cc:1367] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.001s
I20240502 01:59:00.086172 16467 raft_consensus.cc:348] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:00.086272 16467 raft_consensus.cc:374] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.086308 16467 raft_consensus.cc:729] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.086400 16466 tablet_bootstrap.cc:492] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:59:00.086467 16466 ts_tablet_manager.cc:1367] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:59:00.086449 16467 consensus_queue.cc:260] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:00.086905 16467 ts_tablet_manager.cc:1398] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:59:00.087069 16467 tablet_bootstrap.cc:492] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:59:00.087105 16466 raft_consensus.cc:348] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.087196 16466 raft_consensus.cc:374] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.087230 16466 raft_consensus.cc:729] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.087335 16466 consensus_queue.cc:260] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.087585 16466 ts_tablet_manager.cc:1398] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:59:00.087749 16466 tablet_bootstrap.cc:492] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:59:00.087857 16467 tablet_bootstrap.cc:654] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.088508 16466 tablet_bootstrap.cc:654] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.089332 16466 tablet_bootstrap.cc:492] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:59:00.089352 16467 tablet_bootstrap.cc:492] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:59:00.089403 16466 ts_tablet_manager.cc:1367] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 01:59:00.089413 16467 ts_tablet_manager.cc:1367] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 01:59:00.089707 16466 raft_consensus.cc:348] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.089735 16467 raft_consensus.cc:348] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.089787 16466 raft_consensus.cc:374] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.089809 16467 raft_consensus.cc:374] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.089821 16466 raft_consensus.cc:729] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.089843 16467 raft_consensus.cc:729] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.089938 16467 consensus_queue.cc:260] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.089924 16466 consensus_queue.cc:260] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.090173 16466 ts_tablet_manager.cc:1398] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:00.090168 16467 ts_tablet_manager.cc:1398] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:59:00.094923 16468 raft_consensus.cc:348] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:00.095016 16468 raft_consensus.cc:374] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.095045 16468 raft_consensus.cc:729] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.095295 16468 consensus_queue.cc:260] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:00.095604 16468 ts_tablet_manager.cc:1398] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.011s user 0.001s sys 0.000s
I20240502 01:59:00.096045 16468 tablet_bootstrap.cc:492] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:59:00.096984 16468 tablet_bootstrap.cc:654] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.099133 16468 tablet_bootstrap.cc:492] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:59:00.099210 16468 ts_tablet_manager.cc:1367] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:59:00.099550 16468 raft_consensus.cc:348] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.099639 16468 raft_consensus.cc:374] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.099673 16468 raft_consensus.cc:729] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.099763 16468 consensus_queue.cc:260] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.099967 16468 ts_tablet_manager.cc:1398] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:00.100116 16468 tablet_bootstrap.cc:492] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:59:00.100819 16468 tablet_bootstrap.cc:654] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.101728 16468 tablet_bootstrap.cc:492] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:59:00.101821 16468 ts_tablet_manager.cc:1367] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:59:00.102162 16468 raft_consensus.cc:348] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.102227 16468 raft_consensus.cc:374] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.102246 16468 raft_consensus.cc:729] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.102319 16468 consensus_queue.cc:260] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.102491 16468 ts_tablet_manager.cc:1398] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:00.135885 16216 raft_consensus.cc:482] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:00.135988 16216 raft_consensus.cc:504] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:00.136401 16216 leader_election.cc:288] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:00.136718 15195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "490e37fac0844a08a07a4b281bc33bf9" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" is_pre_election: true
I20240502 01:59:00.136819 15466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "490e37fac0844a08a07a4b281bc33bf9" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:59:00.136874 15195 raft_consensus.cc:2454] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 0.
I20240502 01:59:00.136940 15466 raft_consensus.cc:2454] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 0.
I20240502 01:59:00.137132 15264 leader_election.cc:302] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:59:00.137322 16216 raft_consensus.cc:2781] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:59:00.137377 16216 raft_consensus.cc:482] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:59:00.137423 16216 raft_consensus.cc:3037] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.138152 16216 raft_consensus.cc:504] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:00.138453 16216 leader_election.cc:288] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 election: Requested vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:00.138790 15466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "490e37fac0844a08a07a4b281bc33bf9" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:59:00.138813 15195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "490e37fac0844a08a07a4b281bc33bf9" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:59:00.138895 15466 raft_consensus.cc:3037] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.138902 15195 raft_consensus.cc:3037] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.139578 15466 raft_consensus.cc:2454] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 1.
I20240502 01:59:00.139588 15195 raft_consensus.cc:2454] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 1.
I20240502 01:59:00.139856 15265 leader_election.cc:302] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:59:00.140056 16216 raft_consensus.cc:2781] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:59:00.140123 16216 raft_consensus.cc:686] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Becoming Leader. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Running, Role: LEADER
I20240502 01:59:00.140259 16216 consensus_queue.cc:237] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:00.141824 14908 catalog_manager.cc:5484] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: term changed from 0 to 1, leader changed from <none> to f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67). New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:00.147213 16300 raft_consensus.cc:482] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:00.147307 16300 raft_consensus.cc:504] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.147660 16300 leader_election.cc:288] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:00.147964 15195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ac5ec2c673cb4b988d1099e0442ec63f" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" is_pre_election: true
I20240502 01:59:00.147977 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ac5ec2c673cb4b988d1099e0442ec63f" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:59:00.148067 15330 raft_consensus.cc:2454] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 0.
I20240502 01:59:00.148066 15195 raft_consensus.cc:2454] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 0.
I20240502 01:59:00.148305 15397 leader_election.cc:302] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5; no voters:
I20240502 01:59:00.148456 16300 raft_consensus.cc:2781] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:59:00.148509 16300 raft_consensus.cc:482] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:59:00.148535 16300 raft_consensus.cc:3037] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.149236 16300 raft_consensus.cc:504] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.149510 16300 leader_election.cc:288] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 election: Requested vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:00.149814 15195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ac5ec2c673cb4b988d1099e0442ec63f" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:59:00.149843 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ac5ec2c673cb4b988d1099e0442ec63f" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:59:00.149914 15195 raft_consensus.cc:3037] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.149956 15330 raft_consensus.cc:3037] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.150641 15195 raft_consensus.cc:2454] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 1.
I20240502 01:59:00.150933 15397 leader_election.cc:302] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5; no voters:
I20240502 01:59:00.151103 16300 raft_consensus.cc:2781] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:59:00.151309 16300 raft_consensus.cc:686] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Becoming Leader. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Running, Role: LEADER
I20240502 01:59:00.151458 16300 consensus_queue.cc:237] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.152891 14908 catalog_manager.cc:5484] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:00.167236 15330 raft_consensus.cc:2454] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 1.
I20240502 01:59:00.316097 16223 raft_consensus.cc:482] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:00.316226 16223 raft_consensus.cc:504] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.316625 16223 leader_election.cc:288] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:00.317024 15195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d1c832c721fb4b15bc6d266a1c295401" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" is_pre_election: true
I20240502 01:59:00.317160 15195 raft_consensus.cc:2454] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 0.
I20240502 01:59:00.317122 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d1c832c721fb4b15bc6d266a1c295401" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:59:00.317255 15330 raft_consensus.cc:2454] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 0.
I20240502 01:59:00.317448 14993 leader_election.cc:302] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, ddae2a636423489693c12b7d7d166c09; no voters:
I20240502 01:59:00.317608 16223 raft_consensus.cc:2781] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:59:00.317663 16223 raft_consensus.cc:482] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:59:00.317687 16223 raft_consensus.cc:3037] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.318432 16223 raft_consensus.cc:504] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.318811 16223 leader_election.cc:288] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 election: Requested vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:00.319141 15195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d1c832c721fb4b15bc6d266a1c295401" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:59:00.319175 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d1c832c721fb4b15bc6d266a1c295401" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:59:00.319243 15195 raft_consensus.cc:3037] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.319272 15330 raft_consensus.cc:3037] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.319981 15195 raft_consensus.cc:2454] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 1.
I20240502 01:59:00.319975 15330 raft_consensus.cc:2454] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 1.
I20240502 01:59:00.320277 14993 leader_election.cc:302] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:59:00.320475 16223 raft_consensus.cc:2781] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:59:00.320571 16223 raft_consensus.cc:686] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Becoming Leader. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Running, Role: LEADER
I20240502 01:59:00.320706 16223 consensus_queue.cc:237] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.322484 14908 catalog_manager.cc:5484] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 reported cstate change: term changed from 0 to 1, leader changed from <none> to ddae2a636423489693c12b7d7d166c09 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } }
W20240502 01:59:00.472981 16483 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:59:00.473196 16483 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:59:00.476227 16483 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_5'
I20240502 01:59:00.535086 16216 consensus_queue.cc:1035] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:00.538906 16386 consensus_queue.cc:1035] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:00.553867 15330 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:00.553917 15331 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:00.553922 15197 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:00.554224 15195 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:00.554335 16387 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:00.554416 16300 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:00.554687 16300 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:00.555111 16387 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 103.853 ms
time per row: 0.0519264 ms
I20240502 01:59:00.620709 14915 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54458:
name: "rebalance_test_table_6"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:59:00.625744 15446 tablet_service.cc:1467] Processing CreateTablet for tablet 81c8fe21ec7545e79450532b43a61ee7 (DEFAULT_TABLE table=rebalance_test_table_6 [id=2739aac0519d438587e8ec91530f5579]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.625759 15040 tablet_service.cc:1467] Processing CreateTablet for tablet 81c8fe21ec7545e79450532b43a61ee7 (DEFAULT_TABLE table=rebalance_test_table_6 [id=2739aac0519d438587e8ec91530f5579]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.626039 15446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81c8fe21ec7545e79450532b43a61ee7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.626071 15040 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81c8fe21ec7545e79450532b43a61ee7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.626895 15174 tablet_service.cc:1467] Processing CreateTablet for tablet 81c8fe21ec7545e79450532b43a61ee7 (DEFAULT_TABLE table=rebalance_test_table_6 [id=2739aac0519d438587e8ec91530f5579]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.627146 15174 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81c8fe21ec7545e79450532b43a61ee7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.627213 15041 tablet_service.cc:1467] Processing CreateTablet for tablet 6d5cffb7fcc54118861463e3b1fa4a8d (DEFAULT_TABLE table=rebalance_test_table_6 [id=2739aac0519d438587e8ec91530f5579]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.627213 15039 tablet_service.cc:1467] Processing CreateTablet for tablet 8f2fb504a49c44edac92fd935555c278 (DEFAULT_TABLE table=rebalance_test_table_6 [id=2739aac0519d438587e8ec91530f5579]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.627422 15041 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6d5cffb7fcc54118861463e3b1fa4a8d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.627583 15039 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8f2fb504a49c44edac92fd935555c278. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.627924 15445 tablet_service.cc:1467] Processing CreateTablet for tablet 6d5cffb7fcc54118861463e3b1fa4a8d (DEFAULT_TABLE table=rebalance_test_table_6 [id=2739aac0519d438587e8ec91530f5579]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.628119 15445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6d5cffb7fcc54118861463e3b1fa4a8d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.629410 15309 tablet_service.cc:1467] Processing CreateTablet for tablet 6d5cffb7fcc54118861463e3b1fa4a8d (DEFAULT_TABLE table=rebalance_test_table_6 [id=2739aac0519d438587e8ec91530f5579]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.629660 15309 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6d5cffb7fcc54118861463e3b1fa4a8d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.629889 16551 tablet_bootstrap.cc:492] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:59:00.630036 15444 tablet_service.cc:1467] Processing CreateTablet for tablet 8f2fb504a49c44edac92fd935555c278 (DEFAULT_TABLE table=rebalance_test_table_6 [id=2739aac0519d438587e8ec91530f5579]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.630242 15444 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8f2fb504a49c44edac92fd935555c278. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.630895 16551 tablet_bootstrap.cc:654] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.631947 16551 tablet_bootstrap.cc:492] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:59:00.632026 16551 ts_tablet_manager.cc:1367] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:59:00.626895 15175 tablet_service.cc:1467] Processing CreateTablet for tablet 8f2fb504a49c44edac92fd935555c278 (DEFAULT_TABLE table=rebalance_test_table_6 [id=2739aac0519d438587e8ec91530f5579]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:00.632433 15175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8f2fb504a49c44edac92fd935555c278. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:00.632455 16551 raft_consensus.cc:348] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.632539 16551 raft_consensus.cc:374] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.632565 16551 raft_consensus.cc:729] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.632678 16551 consensus_queue.cc:260] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.632927 16551 ts_tablet_manager.cc:1398] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:00.633075 16551 tablet_bootstrap.cc:492] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:59:00.633838 16551 tablet_bootstrap.cc:654] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.634411 16549 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:59:00.635282 16549 tablet_bootstrap.cc:654] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.635823 16551 tablet_bootstrap.cc:492] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:59:00.635910 16551 ts_tablet_manager.cc:1367] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:59:00.636366 16553 tablet_bootstrap.cc:492] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:59:00.636309 16551 raft_consensus.cc:348] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.636405 16551 raft_consensus.cc:374] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.636435 16551 raft_consensus.cc:729] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.636550 16551 consensus_queue.cc:260] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.636830 16551 ts_tablet_manager.cc:1398] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:00.636976 16551 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:59:00.637228 16553 tablet_bootstrap.cc:654] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.637914 16549 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:59:00.637951 16551 tablet_bootstrap.cc:654] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.637993 16549 ts_tablet_manager.cc:1367] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:59:00.638382 16549 raft_consensus.cc:348] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.638482 16549 raft_consensus.cc:374] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.638521 16549 raft_consensus.cc:729] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.638651 16549 consensus_queue.cc:260] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.638918 16551 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:59:00.638922 16549 ts_tablet_manager.cc:1398] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:00.638985 16551 ts_tablet_manager.cc:1367] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:59:00.637796 16550 tablet_bootstrap.cc:492] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:59:00.639065 16549 tablet_bootstrap.cc:492] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:59:00.639305 16551 raft_consensus.cc:348] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.639382 16551 raft_consensus.cc:374] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.639411 16551 raft_consensus.cc:729] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.639498 16551 consensus_queue.cc:260] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.639724 16551 ts_tablet_manager.cc:1398] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:00.639849 16549 tablet_bootstrap.cc:654] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.639855 16550 tablet_bootstrap.cc:654] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.640961 16553 tablet_bootstrap.cc:492] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:59:00.641263 16553 ts_tablet_manager.cc:1367] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 01:59:00.641613 16549 tablet_bootstrap.cc:492] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:59:00.641685 16549 ts_tablet_manager.cc:1367] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:59:00.641645 16553 raft_consensus.cc:348] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.641732 16553 raft_consensus.cc:374] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.641762 16553 raft_consensus.cc:729] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.641909 16553 consensus_queue.cc:260] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.642099 16549 raft_consensus.cc:348] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.642160 16553 ts_tablet_manager.cc:1398] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:00.642199 16549 raft_consensus.cc:374] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.642228 16549 raft_consensus.cc:729] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.642355 16549 consensus_queue.cc:260] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.642578 16550 tablet_bootstrap.cc:492] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:59:00.642657 16549 ts_tablet_manager.cc:1398] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:00.642817 16550 ts_tablet_manager.cc:1367] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 01:59:00.643234 16550 raft_consensus.cc:348] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.643332 16550 raft_consensus.cc:374] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.643358 16550 raft_consensus.cc:729] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.643482 16550 consensus_queue.cc:260] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.643725 16550 ts_tablet_manager.cc:1398] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:59:00.643873 16550 tablet_bootstrap.cc:492] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:59:00.644658 16550 tablet_bootstrap.cc:654] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.647017 16550 tablet_bootstrap.cc:492] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:59:00.647104 16550 ts_tablet_manager.cc:1367] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.001s
I20240502 01:59:00.647490 16550 raft_consensus.cc:348] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.647603 16550 raft_consensus.cc:374] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.647634 16550 raft_consensus.cc:729] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.647768 16550 consensus_queue.cc:260] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.648054 16550 ts_tablet_manager.cc:1398] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:59:00.648228 16550 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:59:00.649220 16550 tablet_bootstrap.cc:654] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:00.650700 16550 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:59:00.650789 16550 ts_tablet_manager.cc:1367] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.001s
I20240502 01:59:00.651161 16550 raft_consensus.cc:348] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.651255 16550 raft_consensus.cc:374] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:00.651289 16550 raft_consensus.cc:729] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:59:00.651402 16550 consensus_queue.cc:260] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.651643 16550 ts_tablet_manager.cc:1398] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:00.670673 16524 raft_consensus.cc:482] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:00.670814 16524 raft_consensus.cc:504] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.671191 16524 leader_election.cc:288] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:00.671454 15060 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "81c8fe21ec7545e79450532b43a61ee7" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09" is_pre_election: true
I20240502 01:59:00.671582 15060 raft_consensus.cc:2454] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:59:00.671593 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "81c8fe21ec7545e79450532b43a61ee7" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:59:00.671753 15465 raft_consensus.cc:2454] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:59:00.671875 15130 leader_election.cc:302] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, ddae2a636423489693c12b7d7d166c09; no voters:
I20240502 01:59:00.672042 16524 raft_consensus.cc:2781] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:59:00.672087 16524 raft_consensus.cc:482] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:59:00.672112 16524 raft_consensus.cc:3037] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.672922 16524 raft_consensus.cc:504] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.673173 16524 leader_election.cc:288] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Requested vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:00.673485 15060 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "81c8fe21ec7545e79450532b43a61ee7" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:59:00.673555 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "81c8fe21ec7545e79450532b43a61ee7" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:59:00.673599 15060 raft_consensus.cc:3037] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.673661 15465 raft_consensus.cc:3037] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.674528 15465 raft_consensus.cc:2454] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:59:00.674528 15060 raft_consensus.cc:2454] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:59:00.674813 15130 leader_election.cc:302] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, ddae2a636423489693c12b7d7d166c09; no voters:
I20240502 01:59:00.674969 16524 raft_consensus.cc:2781] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:59:00.675065 16524 raft_consensus.cc:686] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Becoming Leader. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Running, Role: LEADER
I20240502 01:59:00.675197 16524 consensus_queue.cc:237] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.677129 14912 catalog_manager.cc:5484] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: term changed from 0 to 1, leader changed from <none> to 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66). New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } }
I20240502 01:59:00.776645 16223 raft_consensus.cc:482] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:00.776801 16223 raft_consensus.cc:504] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.777208 16223 leader_election.cc:288] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:00.777442 16300 raft_consensus.cc:482] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:00.777550 16300 raft_consensus.cc:504] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.777626 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8f2fb504a49c44edac92fd935555c278" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:59:00.777649 15195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8f2fb504a49c44edac92fd935555c278" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" is_pre_election: true
I20240502 01:59:00.777706 15465 raft_consensus.cc:2454] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 0.
I20240502 01:59:00.777758 15195 raft_consensus.cc:2454] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 0.
I20240502 01:59:00.777865 16300 leader_election.cc:288] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:00.778049 14994 leader_election.cc:302] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, ddae2a636423489693c12b7d7d166c09; no voters:
I20240502 01:59:00.778254 16223 raft_consensus.cc:2781] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:59:00.778234 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6d5cffb7fcc54118861463e3b1fa4a8d" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:59:00.778307 16223 raft_consensus.cc:482] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:59:00.778263 15060 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6d5cffb7fcc54118861463e3b1fa4a8d" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09" is_pre_election: true
I20240502 01:59:00.778360 16223 raft_consensus.cc:3037] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.778365 15330 raft_consensus.cc:2454] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 0.
I20240502 01:59:00.778401 15060 raft_consensus.cc:2454] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 0.
I20240502 01:59:00.778659 15398 leader_election.cc:302] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, ddae2a636423489693c12b7d7d166c09; no voters:
I20240502 01:59:00.778800 16300 raft_consensus.cc:2781] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:59:00.778853 16300 raft_consensus.cc:482] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:59:00.778887 16300 raft_consensus.cc:3037] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.779310 16223 raft_consensus.cc:504] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.779594 16223 leader_election.cc:288] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 election: Requested vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:00.779624 16300 raft_consensus.cc:504] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.779892 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8f2fb504a49c44edac92fd935555c278" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:59:00.779937 15195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8f2fb504a49c44edac92fd935555c278" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:59:00.779994 16300 leader_election.cc:288] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 election: Requested vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:00.780025 15465 raft_consensus.cc:3037] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.780045 15195 raft_consensus.cc:3037] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.780263 15060 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6d5cffb7fcc54118861463e3b1fa4a8d" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:59:00.780356 15060 raft_consensus.cc:3037] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.780364 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6d5cffb7fcc54118861463e3b1fa4a8d" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:59:00.780457 15330 raft_consensus.cc:3037] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:00.780875 15465 raft_consensus.cc:2454] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 1.
I20240502 01:59:00.780875 15195 raft_consensus.cc:2454] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 1.
I20240502 01:59:00.781190 14994 leader_election.cc:302] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, ddae2a636423489693c12b7d7d166c09; no voters:
I20240502 01:59:00.781198 15060 raft_consensus.cc:2454] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 1.
I20240502 01:59:00.781337 15330 raft_consensus.cc:2454] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 1.
I20240502 01:59:00.781447 16223 raft_consensus.cc:2781] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:59:00.781543 15398 leader_election.cc:302] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, ddae2a636423489693c12b7d7d166c09; no voters:
I20240502 01:59:00.781562 16223 raft_consensus.cc:686] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Becoming Leader. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Running, Role: LEADER
I20240502 01:59:00.781733 16300 raft_consensus.cc:2781] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:59:00.781697 16223 consensus_queue.cc:237] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:00.781877 16300 raft_consensus.cc:686] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Becoming Leader. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Running, Role: LEADER
I20240502 01:59:00.782014 16300 consensus_queue.cc:237] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:00.783738 14912 catalog_manager.cc:5484] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 reported cstate change: term changed from 0 to 1, leader changed from <none> to ddae2a636423489693c12b7d7d166c09 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:00.783738 14906 catalog_manager.cc:5484] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68). New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } }
W20240502 01:59:00.922876 16566 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:59:00.923134 16566 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:59:00.926004 16566 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_6'
I20240502 01:59:01.011121 15465 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.011696 16524 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.012193 15060 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.012301 15195 raft_consensus.cc:1261] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.012674 16524 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.012714 16223 consensus_queue.cc:1035] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.015918 15061 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.016853 15330 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.017299 16480 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.019232 16480 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.020001 16541 mvcc.cc:205] Tried to move back new op lower bound from 7023063617573732352 to 7023063616206815232. Current Snapshot: MvccSnapshot[applied={T|T < 7023063617573732352}]
I20240502 01:59:01.021741 15465 raft_consensus.cc:1261] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.022096 16223 consensus_queue.cc:1035] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.044545 16522 mvcc.cc:205] Tried to move back new op lower bound from 7023063617593004032 to 7023063616644280320. Current Snapshot: MvccSnapshot[applied={T|T < 7023063617593004032}]
INSERT report
rows total: 2000
time total: 115.296 ms
time per row: 0.0576481 ms
I20240502 01:59:01.081281 14909 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54458:
name: "rebalance_test_table_7"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:59:01.086477 15309 tablet_service.cc:1467] Processing CreateTablet for tablet 77ae087802144c45924c1b79f561afe8 (DEFAULT_TABLE table=rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.086802 15309 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 77ae087802144c45924c1b79f561afe8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.088874 16553 tablet_bootstrap.cc:492] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:59:01.089483 15040 tablet_service.cc:1467] Processing CreateTablet for tablet 77ae087802144c45924c1b79f561afe8 (DEFAULT_TABLE table=rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.089531 15041 tablet_service.cc:1467] Processing CreateTablet for tablet e00d1ed5cde044cb9bdebf787590261a (DEFAULT_TABLE table=rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.089628 16553 tablet_bootstrap.cc:654] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.089766 15040 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 77ae087802144c45924c1b79f561afe8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.090174 15309 tablet_service.cc:1467] Processing CreateTablet for tablet e00d1ed5cde044cb9bdebf787590261a (DEFAULT_TABLE table=rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.090318 15041 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e00d1ed5cde044cb9bdebf787590261a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.090380 15309 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e00d1ed5cde044cb9bdebf787590261a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.090864 15444 tablet_service.cc:1467] Processing CreateTablet for tablet 30620f4774b8414eb4e27bd5ecb6ecee (DEFAULT_TABLE table=rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.091060 16553 tablet_bootstrap.cc:492] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:59:01.091123 15444 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 30620f4774b8414eb4e27bd5ecb6ecee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.091133 16553 ts_tablet_manager.cc:1367] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:01.091395 15446 tablet_service.cc:1467] Processing CreateTablet for tablet e00d1ed5cde044cb9bdebf787590261a (DEFAULT_TABLE table=rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.091583 15446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e00d1ed5cde044cb9bdebf787590261a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.091527 16553 raft_consensus.cc:348] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:01.091624 16553 raft_consensus.cc:374] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.091652 16553 raft_consensus.cc:729] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.091768 16553 consensus_queue.cc:260] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:01.092017 16553 ts_tablet_manager.cc:1398] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:01.092061 15175 tablet_service.cc:1467] Processing CreateTablet for tablet 77ae087802144c45924c1b79f561afe8 (DEFAULT_TABLE table=rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.092315 15175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 77ae087802144c45924c1b79f561afe8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.093103 16551 tablet_bootstrap.cc:492] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:59:01.090174 15310 tablet_service.cc:1467] Processing CreateTablet for tablet 30620f4774b8414eb4e27bd5ecb6ecee (DEFAULT_TABLE table=rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.093389 15310 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 30620f4774b8414eb4e27bd5ecb6ecee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.093976 16551 tablet_bootstrap.cc:654] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.094328 16553 tablet_bootstrap.cc:492] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:59:01.094745 15174 tablet_service.cc:1467] Processing CreateTablet for tablet 30620f4774b8414eb4e27bd5ecb6ecee (DEFAULT_TABLE table=rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.094957 15174 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 30620f4774b8414eb4e27bd5ecb6ecee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.095141 16553 tablet_bootstrap.cc:654] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.096196 16553 tablet_bootstrap.cc:492] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:59:01.096293 16553 ts_tablet_manager.cc:1367] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:01.096657 16553 raft_consensus.cc:348] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.096767 16553 raft_consensus.cc:374] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.096781 16550 tablet_bootstrap.cc:492] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:59:01.096817 16553 raft_consensus.cc:729] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.096916 16553 consensus_queue.cc:260] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.097221 16553 ts_tablet_manager.cc:1398] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.097389 16553 tablet_bootstrap.cc:492] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:59:01.097602 16550 tablet_bootstrap.cc:654] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.098035 16553 tablet_bootstrap.cc:654] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.098968 16553 tablet_bootstrap.cc:492] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:59:01.098981 16549 tablet_bootstrap.cc:492] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:59:01.099062 16553 ts_tablet_manager.cc:1367] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:01.099385 16553 raft_consensus.cc:348] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.099463 16553 raft_consensus.cc:374] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.099485 16553 raft_consensus.cc:729] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.099565 16553 consensus_queue.cc:260] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.099730 16549 tablet_bootstrap.cc:654] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.099758 16553 ts_tablet_manager.cc:1398] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.100800 16549 tablet_bootstrap.cc:492] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:59:01.100875 16549 ts_tablet_manager.cc:1367] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:01.101197 16549 raft_consensus.cc:348] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:01.101300 16549 raft_consensus.cc:374] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.101336 16549 raft_consensus.cc:729] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.101435 16549 consensus_queue.cc:260] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:01.101670 16550 tablet_bootstrap.cc:492] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:59:01.101675 16549 ts_tablet_manager.cc:1398] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.101733 16550 ts_tablet_manager.cc:1367] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 01:59:01.101827 16549 tablet_bootstrap.cc:492] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:59:01.102080 16550 raft_consensus.cc:348] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:01.102171 16550 raft_consensus.cc:374] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.102197 16550 raft_consensus.cc:729] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.102286 16550 consensus_queue.cc:260] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:01.102480 16550 ts_tablet_manager.cc:1398] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:01.102648 16550 tablet_bootstrap.cc:492] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:59:01.103384 16550 tablet_bootstrap.cc:654] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.104740 16550 tablet_bootstrap.cc:492] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:59:01.104727 16551 tablet_bootstrap.cc:492] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:59:01.104817 16550 ts_tablet_manager.cc:1367] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:01.104832 16551 ts_tablet_manager.cc:1367] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.012s user 0.002s sys 0.000s
I20240502 01:59:01.105175 16550 raft_consensus.cc:348] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.105262 16550 raft_consensus.cc:374] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.105290 16550 raft_consensus.cc:729] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.105247 16551 raft_consensus.cc:348] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.105347 16551 raft_consensus.cc:374] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.105381 16551 raft_consensus.cc:729] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.105383 16550 consensus_queue.cc:260] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.105515 16549 tablet_bootstrap.cc:654] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.105515 16551 consensus_queue.cc:260] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.105619 16550 ts_tablet_manager.cc:1398] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.105816 16551 ts_tablet_manager.cc:1398] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.106011 16551 tablet_bootstrap.cc:492] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:59:01.106505 16549 tablet_bootstrap.cc:492] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:59:01.106583 16549 ts_tablet_manager.cc:1367] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 01:59:01.107059 16549 raft_consensus.cc:348] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.107164 16549 raft_consensus.cc:374] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.107194 16549 raft_consensus.cc:729] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.107306 16549 consensus_queue.cc:260] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.107555 16549 ts_tablet_manager.cc:1398] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.108925 16551 tablet_bootstrap.cc:654] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.109831 16551 tablet_bootstrap.cc:492] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:59:01.109895 16551 ts_tablet_manager.cc:1367] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 01:59:01.110342 16551 raft_consensus.cc:348] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.110419 16551 raft_consensus.cc:374] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.110437 16551 raft_consensus.cc:729] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.110677 16551 consensus_queue.cc:260] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.111152 16551 ts_tablet_manager.cc:1398] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.118999 16606 raft_consensus.cc:482] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:01.119105 16606 raft_consensus.cc:504] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.119531 16606 leader_election.cc:288] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:01.119787 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e00d1ed5cde044cb9bdebf787590261a" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:59:01.119928 15330 raft_consensus.cc:2454] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 0.
I20240502 01:59:01.120126 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e00d1ed5cde044cb9bdebf787590261a" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:59:01.120188 14993 leader_election.cc:302] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:59:01.120239 15465 raft_consensus.cc:2454] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 0.
I20240502 01:59:01.120329 16606 raft_consensus.cc:2781] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:59:01.120379 16606 raft_consensus.cc:482] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:59:01.120429 16606 raft_consensus.cc:3037] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.121207 16606 raft_consensus.cc:504] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.121512 16606 leader_election.cc:288] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 election: Requested vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:01.121789 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e00d1ed5cde044cb9bdebf787590261a" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:59:01.121789 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e00d1ed5cde044cb9bdebf787590261a" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:59:01.121892 15330 raft_consensus.cc:3037] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.121892 15465 raft_consensus.cc:3037] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.122769 15465 raft_consensus.cc:2454] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 1.
I20240502 01:59:01.122776 15330 raft_consensus.cc:2454] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 1.
I20240502 01:59:01.123037 14993 leader_election.cc:302] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:59:01.123255 16606 raft_consensus.cc:2781] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:59:01.123343 16606 raft_consensus.cc:686] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Becoming Leader. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Running, Role: LEADER
I20240502 01:59:01.123466 16606 consensus_queue.cc:237] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.125066 14913 catalog_manager.cc:5484] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 reported cstate change: term changed from 0 to 1, leader changed from <none> to ddae2a636423489693c12b7d7d166c09 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } }
I20240502 01:59:01.132345 16535 raft_consensus.cc:482] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:01.132439 16535 raft_consensus.cc:504] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.132786 16535 leader_election.cc:288] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:01.133129 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "30620f4774b8414eb4e27bd5ecb6ecee" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:59:01.133139 15195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "30620f4774b8414eb4e27bd5ecb6ecee" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" is_pre_election: true
I20240502 01:59:01.133234 15465 raft_consensus.cc:2454] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 0.
I20240502 01:59:01.133256 15195 raft_consensus.cc:2454] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 0.
I20240502 01:59:01.133492 15265 leader_election.cc:302] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:59:01.133687 16535 raft_consensus.cc:2781] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:59:01.133733 16535 raft_consensus.cc:482] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:59:01.133805 16535 raft_consensus.cc:3037] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.134521 16535 raft_consensus.cc:504] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.134858 16535 leader_election.cc:288] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 election: Requested vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:01.135108 15195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "30620f4774b8414eb4e27bd5ecb6ecee" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:59:01.135234 15195 raft_consensus.cc:3037] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.135206 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "30620f4774b8414eb4e27bd5ecb6ecee" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:59:01.135303 15465 raft_consensus.cc:3037] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.135983 15465 raft_consensus.cc:2454] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 1.
I20240502 01:59:01.135993 15195 raft_consensus.cc:2454] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 1.
I20240502 01:59:01.136229 15264 leader_election.cc:302] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:59:01.136397 16535 raft_consensus.cc:2781] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:59:01.136487 16535 raft_consensus.cc:686] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Becoming Leader. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Running, Role: LEADER
I20240502 01:59:01.136615 16535 consensus_queue.cc:237] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.138170 14913 catalog_manager.cc:5484] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: term changed from 0 to 1, leader changed from <none> to f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67). New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:01.145570 16606 raft_consensus.cc:482] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:01.145661 16606 raft_consensus.cc:504] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:01.145982 16606 leader_election.cc:288] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:01.146304 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "77ae087802144c45924c1b79f561afe8" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:59:01.146327 15195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "77ae087802144c45924c1b79f561afe8" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" is_pre_election: true
I20240502 01:59:01.146435 15195 raft_consensus.cc:2454] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 0.
I20240502 01:59:01.146435 15330 raft_consensus.cc:2454] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 0.
I20240502 01:59:01.146698 14993 leader_election.cc:302] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:59:01.146850 16606 raft_consensus.cc:2781] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:59:01.146903 16606 raft_consensus.cc:482] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:59:01.146927 16606 raft_consensus.cc:3037] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.147619 16606 raft_consensus.cc:504] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:01.147905 16606 leader_election.cc:288] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 election: Requested vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:01.148213 15330 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "77ae087802144c45924c1b79f561afe8" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:59:01.148238 15195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "77ae087802144c45924c1b79f561afe8" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:59:01.148322 15330 raft_consensus.cc:3037] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.148322 15195 raft_consensus.cc:3037] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.149221 15195 raft_consensus.cc:2454] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 1.
I20240502 01:59:01.149304 15330 raft_consensus.cc:2454] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 1.
I20240502 01:59:01.149540 14993 leader_election.cc:302] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:59:01.149719 16606 raft_consensus.cc:2781] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:59:01.149853 16606 raft_consensus.cc:686] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Becoming Leader. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Running, Role: LEADER
I20240502 01:59:01.149982 16606 consensus_queue.cc:237] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } }
I20240502 01:59:01.151628 14913 catalog_manager.cc:5484] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 reported cstate change: term changed from 0 to 1, leader changed from <none> to ddae2a636423489693c12b7d7d166c09 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } }
W20240502 01:59:01.259680 16622 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:59:01.259948 16622 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:59:01.262959 16622 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_7'
I20240502 01:59:01.338675 15465 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.339059 15330 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.339438 16387 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.340600 15195 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.341074 15330 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.342151 16606 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.342311 15195 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.342675 16535 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.343823 15466 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.345458 16535 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.348380 16223 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.348726 16223 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.353567 16529 mvcc.cc:205] Tried to move back new op lower bound from 7023063618934435840 to 7023063618096881664. Current Snapshot: MvccSnapshot[applied={T|T < 7023063618934435840}]
I20240502 01:59:01.357422 16532 mvcc.cc:205] Tried to move back new op lower bound from 7023063618934435840 to 7023063618096881664. Current Snapshot: MvccSnapshot[applied={T|T < 7023063618934435840}]
INSERT report
rows total: 2000
time total: 93.9016 ms
time per row: 0.0469508 ms
I20240502 01:59:01.402849 14906 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54458:
name: "rebalance_test_table_8"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:59:01.408746 15041 tablet_service.cc:1467] Processing CreateTablet for tablet cd00409d47954ee2b7555e7ff09a6e28 (DEFAULT_TABLE table=rebalance_test_table_8 [id=bf7d5082701149bea6b9e5c480f53c10]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.409113 15041 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd00409d47954ee2b7555e7ff09a6e28. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.409117 15446 tablet_service.cc:1467] Processing CreateTablet for tablet d24986c2ebab46f086e3f1f5a53af26b (DEFAULT_TABLE table=rebalance_test_table_8 [id=bf7d5082701149bea6b9e5c480f53c10]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.409350 15446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d24986c2ebab46f086e3f1f5a53af26b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.409914 15309 tablet_service.cc:1467] Processing CreateTablet for tablet cd00409d47954ee2b7555e7ff09a6e28 (DEFAULT_TABLE table=rebalance_test_table_8 [id=bf7d5082701149bea6b9e5c480f53c10]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.409912 15310 tablet_service.cc:1467] Processing CreateTablet for tablet 9c2892024ce942e1bf1834c1b9ad2049 (DEFAULT_TABLE table=rebalance_test_table_8 [id=bf7d5082701149bea6b9e5c480f53c10]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.410233 15309 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd00409d47954ee2b7555e7ff09a6e28. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.410504 15175 tablet_service.cc:1467] Processing CreateTablet for tablet d24986c2ebab46f086e3f1f5a53af26b (DEFAULT_TABLE table=rebalance_test_table_8 [id=bf7d5082701149bea6b9e5c480f53c10]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.410770 15175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d24986c2ebab46f086e3f1f5a53af26b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.411469 16551 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:59:01.411464 16550 tablet_bootstrap.cc:492] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:59:01.412315 16553 tablet_bootstrap.cc:492] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:59:01.412362 16551 tablet_bootstrap.cc:654] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.412362 16550 tablet_bootstrap.cc:654] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.413012 16549 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:59:01.412987 15174 tablet_service.cc:1467] Processing CreateTablet for tablet cd00409d47954ee2b7555e7ff09a6e28 (DEFAULT_TABLE table=rebalance_test_table_8 [id=bf7d5082701149bea6b9e5c480f53c10]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.413123 16553 tablet_bootstrap.cc:654] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.413256 15174 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd00409d47954ee2b7555e7ff09a6e28. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.413625 15041 tablet_service.cc:1467] Processing CreateTablet for tablet d24986c2ebab46f086e3f1f5a53af26b (DEFAULT_TABLE table=rebalance_test_table_8 [id=bf7d5082701149bea6b9e5c480f53c10]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.413863 15041 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d24986c2ebab46f086e3f1f5a53af26b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.413897 16549 tablet_bootstrap.cc:654] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.415011 16550 tablet_bootstrap.cc:492] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:59:01.415127 16550 ts_tablet_manager.cc:1367] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:59:01.415550 15310 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9c2892024ce942e1bf1834c1b9ad2049. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.415594 16550 raft_consensus.cc:348] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.415714 16550 raft_consensus.cc:374] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.415745 16550 raft_consensus.cc:729] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.415869 16550 consensus_queue.cc:260] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.416054 15040 tablet_service.cc:1467] Processing CreateTablet for tablet 9c2892024ce942e1bf1834c1b9ad2049 (DEFAULT_TABLE table=rebalance_test_table_8 [id=bf7d5082701149bea6b9e5c480f53c10]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.416162 16550 ts_tablet_manager.cc:1398] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:01.416321 15040 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9c2892024ce942e1bf1834c1b9ad2049. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.416834 16549 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:59:01.416919 16549 ts_tablet_manager.cc:1367] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 01:59:01.417356 16549 raft_consensus.cc:348] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.417454 16549 raft_consensus.cc:374] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.417484 16549 raft_consensus.cc:729] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.417606 16549 consensus_queue.cc:260] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.417870 16549 ts_tablet_manager.cc:1398] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:01.418064 16549 tablet_bootstrap.cc:492] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:59:01.418855 16550 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:59:01.418962 16549 tablet_bootstrap.cc:654] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.419763 16550 tablet_bootstrap.cc:654] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.420661 16549 tablet_bootstrap.cc:492] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:59:01.420615 15446 tablet_service.cc:1467] Processing CreateTablet for tablet 9c2892024ce942e1bf1834c1b9ad2049 (DEFAULT_TABLE table=rebalance_test_table_8 [id=bf7d5082701149bea6b9e5c480f53c10]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.420740 16549 ts_tablet_manager.cc:1367] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:59:01.420861 16550 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:59:01.420900 15446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9c2892024ce942e1bf1834c1b9ad2049. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.420959 16550 ts_tablet_manager.cc:1367] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:01.421175 16549 raft_consensus.cc:348] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.421262 16549 raft_consensus.cc:374] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.421288 16549 raft_consensus.cc:729] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.421417 16549 consensus_queue.cc:260] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.421393 16550 raft_consensus.cc:348] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.421523 16550 raft_consensus.cc:374] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.421556 16550 raft_consensus.cc:729] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.421674 16550 consensus_queue.cc:260] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.421797 16549 ts_tablet_manager.cc:1398] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.422063 16550 ts_tablet_manager.cc:1398] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.422230 16550 tablet_bootstrap.cc:492] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:59:01.423141 16550 tablet_bootstrap.cc:654] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.424253 16551 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:59:01.424266 16553 tablet_bootstrap.cc:492] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:59:01.424345 16551 ts_tablet_manager.cc:1367] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.013s user 0.001s sys 0.000s
I20240502 01:59:01.424410 16553 ts_tablet_manager.cc:1367] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.012s user 0.001s sys 0.000s
I20240502 01:59:01.424808 16551 raft_consensus.cc:348] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.424914 16551 raft_consensus.cc:374] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.424949 16551 raft_consensus.cc:729] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.424861 16553 raft_consensus.cc:348] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.425016 16553 raft_consensus.cc:374] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.425058 16553 raft_consensus.cc:729] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.425102 16551 consensus_queue.cc:260] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.425202 16553 consensus_queue.cc:260] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.425436 16551 ts_tablet_manager.cc:1398] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.425499 16553 ts_tablet_manager.cc:1398] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:01.425621 16551 tablet_bootstrap.cc:492] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:59:01.425678 16553 tablet_bootstrap.cc:492] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:59:01.426476 16551 tablet_bootstrap.cc:654] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.426772 16553 tablet_bootstrap.cc:654] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.431465 16551 tablet_bootstrap.cc:492] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:59:01.431555 16551 ts_tablet_manager.cc:1367] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.002s
I20240502 01:59:01.431682 16550 tablet_bootstrap.cc:492] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:59:01.431771 16550 ts_tablet_manager.cc:1367] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.010s user 0.001s sys 0.000s
I20240502 01:59:01.431994 16551 raft_consensus.cc:348] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.432092 16551 raft_consensus.cc:374] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.432132 16551 raft_consensus.cc:729] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.432180 16550 raft_consensus.cc:348] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.432267 16550 raft_consensus.cc:374] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.432296 16550 raft_consensus.cc:729] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.432252 16551 consensus_queue.cc:260] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.432534 16551 ts_tablet_manager.cc:1398] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:01.432401 16550 consensus_queue.cc:260] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.434087 16550 ts_tablet_manager.cc:1398] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:01.434854 16553 tablet_bootstrap.cc:492] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:59:01.435196 16553 ts_tablet_manager.cc:1367] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.000s
I20240502 01:59:01.435617 16553 raft_consensus.cc:348] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.435734 16553 raft_consensus.cc:374] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.435770 16553 raft_consensus.cc:729] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.435873 16553 consensus_queue.cc:260] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.436151 16553 ts_tablet_manager.cc:1398] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.446496 16607 raft_consensus.cc:482] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:01.446614 16607 raft_consensus.cc:504] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.447016 16607 leader_election.cc:288] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:01.447368 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd00409d47954ee2b7555e7ff09a6e28" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:59:01.447367 15060 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd00409d47954ee2b7555e7ff09a6e28" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09" is_pre_election: true
I20240502 01:59:01.447492 15060 raft_consensus.cc:2454] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:59:01.447491 15329 raft_consensus.cc:2454] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:59:01.447767 15129 leader_election.cc:302] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:59:01.448012 16607 raft_consensus.cc:2781] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:59:01.448062 16607 raft_consensus.cc:482] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:59:01.448112 16607 raft_consensus.cc:3037] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.449038 16607 raft_consensus.cc:504] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.449330 16607 leader_election.cc:288] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Requested vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:01.449992 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd00409d47954ee2b7555e7ff09a6e28" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:59:01.450110 15329 raft_consensus.cc:3037] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.450237 15060 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd00409d47954ee2b7555e7ff09a6e28" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:59:01.450332 15060 raft_consensus.cc:3037] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.451022 15329 raft_consensus.cc:2454] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:59:01.451251 15060 raft_consensus.cc:2454] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:59:01.451308 15129 leader_election.cc:302] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:59:01.451469 16607 raft_consensus.cc:2781] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:59:01.451577 16607 raft_consensus.cc:686] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Becoming Leader. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Running, Role: LEADER
I20240502 01:59:01.451682 16607 consensus_queue.cc:237] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.453346 14915 catalog_manager.cc:5484] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: term changed from 0 to 1, leader changed from <none> to 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66). New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:01.471343 16535 raft_consensus.cc:482] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:01.471423 16535 raft_consensus.cc:504] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.471699 16535 leader_election.cc:288] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:01.471977 15060 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c2892024ce942e1bf1834c1b9ad2049" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09" is_pre_election: true
I20240502 01:59:01.472103 15060 raft_consensus.cc:2454] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 0.
I20240502 01:59:01.472082 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c2892024ce942e1bf1834c1b9ad2049" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:59:01.472199 15465 raft_consensus.cc:2454] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 0.
I20240502 01:59:01.472335 15265 leader_election.cc:302] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:59:01.472501 16535 raft_consensus.cc:2781] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:59:01.472553 16535 raft_consensus.cc:482] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:59:01.472599 16535 raft_consensus.cc:3037] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.473332 16535 raft_consensus.cc:504] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.473627 16535 leader_election.cc:288] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 election: Requested vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:01.473910 15060 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c2892024ce942e1bf1834c1b9ad2049" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:59:01.473948 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c2892024ce942e1bf1834c1b9ad2049" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:59:01.474040 15060 raft_consensus.cc:3037] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.474038 15465 raft_consensus.cc:3037] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.474916 15060 raft_consensus.cc:2454] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 1.
I20240502 01:59:01.474916 15465 raft_consensus.cc:2454] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 1.
I20240502 01:59:01.475154 15265 leader_election.cc:302] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:59:01.475311 16535 raft_consensus.cc:2781] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:59:01.475394 16535 raft_consensus.cc:686] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Becoming Leader. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Running, Role: LEADER
I20240502 01:59:01.475549 16535 consensus_queue.cc:237] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }
I20240502 01:59:01.477164 14915 catalog_manager.cc:5484] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: term changed from 0 to 1, leader changed from <none> to f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67). New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:01.492962 16607 raft_consensus.cc:482] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:01.493088 16607 raft_consensus.cc:504] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.493415 16607 leader_election.cc:288] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:01.493752 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d24986c2ebab46f086e3f1f5a53af26b" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:59:01.493788 15060 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d24986c2ebab46f086e3f1f5a53af26b" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09" is_pre_election: true
I20240502 01:59:01.493860 15465 raft_consensus.cc:2454] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:59:01.493939 15060 raft_consensus.cc:2454] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:59:01.494148 15130 leader_election.cc:302] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5; no voters:
I20240502 01:59:01.494338 16607 raft_consensus.cc:2781] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:59:01.494403 16607 raft_consensus.cc:482] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:59:01.494436 16607 raft_consensus.cc:3037] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.495170 16607 raft_consensus.cc:504] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.495450 16607 leader_election.cc:288] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Requested vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:01.495749 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d24986c2ebab46f086e3f1f5a53af26b" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:59:01.495846 15060 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d24986c2ebab46f086e3f1f5a53af26b" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:59:01.495869 15465 raft_consensus.cc:3037] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.495944 15060 raft_consensus.cc:3037] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.496776 15060 raft_consensus.cc:2454] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:59:01.496776 15465 raft_consensus.cc:2454] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:59:01.497053 15130 leader_election.cc:302] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, ddae2a636423489693c12b7d7d166c09; no voters:
I20240502 01:59:01.497193 16607 raft_consensus.cc:2781] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:59:01.497277 16607 raft_consensus.cc:686] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Becoming Leader. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Running, Role: LEADER
I20240502 01:59:01.497403 16607 consensus_queue.cc:237] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 01:59:01.499158 14915 catalog_manager.cc:5484] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: term changed from 0 to 1, leader changed from <none> to 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66). New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } }
W20240502 01:59:01.639482 16675 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:59:01.639724 16675 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:59:01.642573 16675 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_8'
I20240502 01:59:01.715283 15061 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.715284 15329 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.715987 15465 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.716181 16314 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.716511 16314 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.717495 15060 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.720847 15465 raft_consensus.cc:1261] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.721115 16314 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.721341 16607 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.721508 16535 consensus_queue.cc:1035] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:01.725998 15059 raft_consensus.cc:1261] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:01.726467 16659 consensus_queue.cc:1035] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 97.6511 ms
time per row: 0.0488256 ms
I20240502 01:59:01.781234 14915 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54458:
name: "rebalance_test_table_9"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 01:59:01.786746 15446 tablet_service.cc:1467] Processing CreateTablet for tablet 856f623374824d94913db0cb9faa25d1 (DEFAULT_TABLE table=rebalance_test_table_9 [id=12ee6dc62f814bd7b3c13bee4495d9e0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.786839 15174 tablet_service.cc:1467] Processing CreateTablet for tablet 856f623374824d94913db0cb9faa25d1 (DEFAULT_TABLE table=rebalance_test_table_9 [id=12ee6dc62f814bd7b3c13bee4495d9e0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.787084 15174 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 856f623374824d94913db0cb9faa25d1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.787076 15446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 856f623374824d94913db0cb9faa25d1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.788575 16549 tablet_bootstrap.cc:492] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:59:01.788838 16551 tablet_bootstrap.cc:492] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:59:01.789408 16549 tablet_bootstrap.cc:654] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.789547 15310 tablet_service.cc:1467] Processing CreateTablet for tablet 856f623374824d94913db0cb9faa25d1 (DEFAULT_TABLE table=rebalance_test_table_9 [id=12ee6dc62f814bd7b3c13bee4495d9e0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.789682 16551 tablet_bootstrap.cc:654] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.789726 15311 tablet_service.cc:1467] Processing CreateTablet for tablet 6c4e85e7050b4dc8a4ce6bba818854a2 (DEFAULT_TABLE table=rebalance_test_table_9 [id=12ee6dc62f814bd7b3c13bee4495d9e0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.789810 15310 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 856f623374824d94913db0cb9faa25d1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.789896 15311 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6c4e85e7050b4dc8a4ce6bba818854a2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.789547 15309 tablet_service.cc:1467] Processing CreateTablet for tablet a901eb3063ea4911a822552db01d365d (DEFAULT_TABLE table=rebalance_test_table_9 [id=12ee6dc62f814bd7b3c13bee4495d9e0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.790323 15309 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a901eb3063ea4911a822552db01d365d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.791635 15040 tablet_service.cc:1467] Processing CreateTablet for tablet 6c4e85e7050b4dc8a4ce6bba818854a2 (DEFAULT_TABLE table=rebalance_test_table_9 [id=12ee6dc62f814bd7b3c13bee4495d9e0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.791791 16553 tablet_bootstrap.cc:492] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:59:01.791756 15175 tablet_service.cc:1467] Processing CreateTablet for tablet 6c4e85e7050b4dc8a4ce6bba818854a2 (DEFAULT_TABLE table=rebalance_test_table_9 [id=12ee6dc62f814bd7b3c13bee4495d9e0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.791899 15040 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6c4e85e7050b4dc8a4ce6bba818854a2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.791970 15175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6c4e85e7050b4dc8a4ce6bba818854a2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.792578 16553 tablet_bootstrap.cc:654] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.792663 15174 tablet_service.cc:1467] Processing CreateTablet for tablet a901eb3063ea4911a822552db01d365d (DEFAULT_TABLE table=rebalance_test_table_9 [id=12ee6dc62f814bd7b3c13bee4495d9e0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.792889 15174 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a901eb3063ea4911a822552db01d365d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.793561 15446 tablet_service.cc:1467] Processing CreateTablet for tablet a901eb3063ea4911a822552db01d365d (DEFAULT_TABLE table=rebalance_test_table_9 [id=12ee6dc62f814bd7b3c13bee4495d9e0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 01:59:01.793799 15446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a901eb3063ea4911a822552db01d365d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:01.794026 16550 tablet_bootstrap.cc:492] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:59:01.794806 16550 tablet_bootstrap.cc:654] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.795179 16549 tablet_bootstrap.cc:492] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:59:01.795260 16549 ts_tablet_manager.cc:1367] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 01:59:01.795696 16549 raft_consensus.cc:348] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.795809 16550 tablet_bootstrap.cc:492] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09: No bootstrap required, opened a new log
I20240502 01:59:01.795812 16549 raft_consensus.cc:374] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.795848 16549 raft_consensus.cc:729] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.795889 16550 ts_tablet_manager.cc:1367] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:01.795975 16549 consensus_queue.cc:260] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.796278 16549 ts_tablet_manager.cc:1398] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.796337 16550 raft_consensus.cc:348] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.796455 16550 raft_consensus.cc:374] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.796484 16549 tablet_bootstrap.cc:492] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:59:01.796485 16550 raft_consensus.cc:729] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.796600 16550 consensus_queue.cc:260] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.796897 16550 ts_tablet_manager.cc:1398] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.796969 16553 tablet_bootstrap.cc:492] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:59:01.797044 16553 ts_tablet_manager.cc:1367] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 01:59:01.797303 16549 tablet_bootstrap.cc:654] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.797353 16553 raft_consensus.cc:348] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.797417 16553 raft_consensus.cc:374] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.797435 16553 raft_consensus.cc:729] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.797541 16553 consensus_queue.cc:260] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.797806 16553 ts_tablet_manager.cc:1398] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:01.797931 16553 tablet_bootstrap.cc:492] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:59:01.798512 16553 tablet_bootstrap.cc:654] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.799554 16549 tablet_bootstrap.cc:492] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:59:01.799610 16551 tablet_bootstrap.cc:492] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:59:01.799631 16549 ts_tablet_manager.cc:1367] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:59:01.799719 16551 ts_tablet_manager.cc:1367] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.002s
I20240502 01:59:01.799984 16549 raft_consensus.cc:348] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.800067 16549 raft_consensus.cc:374] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.800096 16549 raft_consensus.cc:729] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.800179 16551 raft_consensus.cc:348] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.800230 16549 consensus_queue.cc:260] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.800290 16551 raft_consensus.cc:374] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.800339 16551 raft_consensus.cc:729] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.800474 16551 consensus_queue.cc:260] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.800522 16549 ts_tablet_manager.cc:1398] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.800799 16551 ts_tablet_manager.cc:1398] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:01.800962 16551 tablet_bootstrap.cc:492] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:59:01.801242 16549 tablet_bootstrap.cc:492] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:59:01.801762 16709 raft_consensus.cc:482] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:01.801827 16551 tablet_bootstrap.cc:654] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.801854 16709 raft_consensus.cc:504] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.802098 16549 tablet_bootstrap.cc:654] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.802498 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6c4e85e7050b4dc8a4ce6bba818854a2" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09" is_pre_election: true
I20240502 01:59:01.802634 15061 raft_consensus.cc:2454] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:59:01.802778 16709 leader_election.cc:288] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:01.802775 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6c4e85e7050b4dc8a4ce6bba818854a2" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:59:01.802937 15329 raft_consensus.cc:2454] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:59:01.803308 15130 leader_election.cc:302] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, ddae2a636423489693c12b7d7d166c09; no voters:
I20240502 01:59:01.803490 16709 raft_consensus.cc:2781] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:59:01.803546 16709 raft_consensus.cc:482] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:59:01.803581 16709 raft_consensus.cc:3037] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.803666 16553 tablet_bootstrap.cc:492] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:59:01.803755 16553 ts_tablet_manager.cc:1367] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 01:59:01.804201 16553 raft_consensus.cc:348] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.804291 16553 raft_consensus.cc:374] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.804338 16553 raft_consensus.cc:729] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.804334 16551 tablet_bootstrap.cc:492] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: No bootstrap required, opened a new log
I20240502 01:59:01.804432 16551 ts_tablet_manager.cc:1367] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 01:59:01.804462 16553 consensus_queue.cc:260] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.804540 16709 raft_consensus.cc:504] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.804713 16553 ts_tablet_manager.cc:1398] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.804905 16553 tablet_bootstrap.cc:492] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:59:01.804862 16551 raft_consensus.cc:348] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.804955 16551 raft_consensus.cc:374] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.804987 16551 raft_consensus.cc:729] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.805112 15061 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6c4e85e7050b4dc8a4ce6bba818854a2" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:59:01.805094 16551 consensus_queue.cc:260] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.805197 15061 raft_consensus.cc:3037] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.805402 16551 ts_tablet_manager.cc:1398] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:01.805415 16549 tablet_bootstrap.cc:492] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b: No bootstrap required, opened a new log
I20240502 01:59:01.805486 16549 ts_tablet_manager.cc:1367] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 01:59:01.805792 16553 tablet_bootstrap.cc:654] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171: Neither blocks nor log segments found. Creating new log.
I20240502 01:59:01.805931 16549 raft_consensus.cc:348] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.806027 16549 raft_consensus.cc:374] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.806058 16549 raft_consensus.cc:729] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.806073 15061 raft_consensus.cc:2454] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:59:01.806174 16549 consensus_queue.cc:260] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.806349 15130 leader_election.cc:302] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, ddae2a636423489693c12b7d7d166c09; no voters:
I20240502 01:59:01.806423 16549 ts_tablet_manager.cc:1398] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:01.806551 16607 raft_consensus.cc:2781] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:59:01.806666 16607 raft_consensus.cc:686] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Becoming Leader. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Running, Role: LEADER
I20240502 01:59:01.806778 16607 consensus_queue.cc:237] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.807504 16709 leader_election.cc:288] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Requested vote from peers ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:01.808116 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6c4e85e7050b4dc8a4ce6bba818854a2" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:59:01.808208 15329 raft_consensus.cc:3037] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.808745 16553 tablet_bootstrap.cc:492] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171: No bootstrap required, opened a new log
I20240502 01:59:01.808815 16553 ts_tablet_manager.cc:1367] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:59:01.809077 15329 raft_consensus.cc:2454] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:59:01.809322 14911 catalog_manager.cc:5484] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: term changed from 0 to 1, leader changed from <none> to 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66). New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:01.809468 16553 raft_consensus.cc:348] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.809571 16553 raft_consensus.cc:374] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 01:59:01.809592 16553 raft_consensus.cc:729] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: FOLLOWER
I20240502 01:59:01.810169 16553 consensus_queue.cc:260] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.810433 16553 ts_tablet_manager.cc:1398] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:01.824692 16708 raft_consensus.cc:482] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:01.824785 16708 raft_consensus.cc:504] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.825058 16708 leader_election.cc:288] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:01.825402 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a901eb3063ea4911a822552db01d365d" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:59:01.825402 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a901eb3063ea4911a822552db01d365d" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:59:01.825517 15329 raft_consensus.cc:2454] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:59:01.825517 15465 raft_consensus.cc:2454] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:59:01.825795 15129 leader_election.cc:302] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:59:01.825982 16708 raft_consensus.cc:2781] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:59:01.826035 16708 raft_consensus.cc:482] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:59:01.826061 16708 raft_consensus.cc:3037] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.826856 16708 raft_consensus.cc:504] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.827139 16708 leader_election.cc:288] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Requested vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:01.827435 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a901eb3063ea4911a822552db01d365d" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:59:01.827468 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a901eb3063ea4911a822552db01d365d" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:59:01.827549 15329 raft_consensus.cc:3037] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.827564 15465 raft_consensus.cc:3037] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.828188 15329 raft_consensus.cc:2454] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:59:01.828271 15465 raft_consensus.cc:2454] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:59:01.828436 15129 leader_election.cc:302] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, f77c2a969bde45b49ac1bbbe242ff171; no voters:
I20240502 01:59:01.828689 16708 raft_consensus.cc:2781] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:59:01.828814 16708 raft_consensus.cc:686] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Becoming Leader. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Running, Role: LEADER
I20240502 01:59:01.828928 16708 consensus_queue.cc:237] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.830673 14912 catalog_manager.cc:5484] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: term changed from 0 to 1, leader changed from <none> to 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66). New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:01.980460 16708 raft_consensus.cc:482] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:01.980574 16708 raft_consensus.cc:504] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.980918 16708 leader_election.cc:288] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:01.981189 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "856f623374824d94913db0cb9faa25d1" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:59:01.981330 15465 raft_consensus.cc:2454] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:59:01.981318 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "856f623374824d94913db0cb9faa25d1" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:59:01.981443 15329 raft_consensus.cc:2454] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 0.
I20240502 01:59:01.981598 15130 leader_election.cc:302] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5; no voters:
I20240502 01:59:01.981794 16708 raft_consensus.cc:2781] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 01:59:01.981840 16708 raft_consensus.cc:482] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 01:59:01.981880 16708 raft_consensus.cc:3037] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.982697 16708 raft_consensus.cc:504] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.982996 16708 leader_election.cc:288] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Requested vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:01.983263 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "856f623374824d94913db0cb9faa25d1" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:59:01.983374 15465 raft_consensus.cc:3037] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.983325 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "856f623374824d94913db0cb9faa25d1" candidate_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:59:01.983453 15329 raft_consensus.cc:3037] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 0 FOLLOWER]: Advancing to term 1
I20240502 01:59:01.984117 15465 raft_consensus.cc:2454] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:59:01.984184 15329 raft_consensus.cc:2454] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26a3d5ced97d4dcc919b80fde53f6a5b in term 1.
I20240502 01:59:01.984406 15130 leader_election.cc:302] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5; no voters:
I20240502 01:59:01.984639 16708 raft_consensus.cc:2781] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 01:59:01.984717 16708 raft_consensus.cc:686] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Becoming Leader. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Running, Role: LEADER
I20240502 01:59:01.984853 16708 consensus_queue.cc:237] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }
I20240502 01:59:01.986557 14912 catalog_manager.cc:5484] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: term changed from 0 to 1, leader changed from <none> to 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66). New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } }
W20240502 01:59:02.135818 16722 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:59:02.136056 16722 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:59:02.138824 16722 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_9'
I20240502 01:59:02.211419 16708 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:02.218362 15465 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:02.218458 15331 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:02.218879 16709 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:02.219198 16709 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:02.219528 15331 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:02.220072 15465 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:02.220415 16709 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:02.223668 16314 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 01:59:02.230798 15061 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 01:59:02.231261 16709 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 111.26 ms
time per row: 0.05563 ms
I20240502 01:59:02.289896 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.69:35223
--local_ip_for_outbound_sockets=127.7.221.69
--tserver_master_addrs=127.7.221.126:37149
--webserver_port=41583
--webserver_interface=127.7.221.69
--builtin_ntp_servers=127.7.221.84:36353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 01:59:02.394979 16756 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:59:02.395213 16756 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:59:02.395284 16756 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:59:02.398175 16756 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:59:02.398242 16756 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:59:02.398346 16756 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.69
I20240502 01:59:02.401908 16756 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.69:35223
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.7.221.69
--webserver_port=41583
--tserver_master_addrs=127.7.221.126:37149
--never_fsync=true
--heap_profile_path=/tmp/kudu.16756
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.69
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:59:02.402794 16756 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:59:02.403849 16756 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:59:02.412948 16761 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:59:02.413849 16756 server_base.cc:1034] running on GCE node
W20240502 01:59:02.414469 16764 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:59:02.415196 16762 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:59:02.415642 16756 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:59:02.416270 16756 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:59:02.417420 16756 hybrid_clock.cc:648] HybridClock initialized: now 1714615142417383 us; error 49 us; skew 500 ppm
I20240502 01:59:02.419399 16756 webserver.cc:460] Webserver started at http://127.7.221.69:41583/ using document root <none> and password file <none>
I20240502 01:59:02.419982 16756 fs_manager.cc:359] Metadata directory not provided
I20240502 01:59:02.420074 16756 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:59:02.423377 16756 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.003s sys 0.000s
I20240502 01:59:02.425225 16770 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:59:02.426330 16756 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
I20240502 01:59:02.426424 16756 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "84987ad9e4a54d96af8569e4aaf0ed64"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:59:02.426913 16756 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:59:02.443605 16756 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:59:02.444178 16756 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:59:02.444335 16756 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:59:02.444887 16756 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:59:02.445876 16756 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:59:02.445931 16756 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:59:02.445976 16756 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:59:02.446003 16756 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:59:02.454649 16756 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.69:35223
I20240502 01:59:02.455634 16756 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 01:59:02.455842 16886 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.69:35223 every 8 connection(s)
I20240502 01:59:02.456362 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 16756
I20240502 01:59:02.456828 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.70:34649
--local_ip_for_outbound_sockets=127.7.221.70
--tserver_master_addrs=127.7.221.126:37149
--webserver_port=33431
--webserver_interface=127.7.221.70
--builtin_ntp_servers=127.7.221.84:36353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:59:02.464916 16887 heartbeater.cc:347] Connected to a master server at 127.7.221.126:37149
I20240502 01:59:02.465129 16887 heartbeater.cc:464] Registering TS with master...
I20240502 01:59:02.465610 16887 heartbeater.cc:511] Master 127.7.221.126:37149 requested a full tablet report, sending...
I20240502 01:59:02.466745 14915 ts_manager.cc:194] Re-registered known tserver with Master: 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223)
I20240502 01:59:02.467308 14915 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.69:55225
W20240502 01:59:02.565596 16889 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:59:02.565826 16889 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:59:02.565893 16889 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:59:02.568801 16889 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:59:02.568878 16889 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:59:02.568989 16889 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.70
I20240502 01:59:02.572305 16889 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.70:34649
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.7.221.70
--webserver_port=33431
--tserver_master_addrs=127.7.221.126:37149
--never_fsync=true
--heap_profile_path=/tmp/kudu.16889
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.70
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:59:02.573194 16889 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:59:02.574373 16889 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:59:02.582947 16895 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:59:02.583796 16896 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:59:02.584501 16898 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:59:02.585351 16889 server_base.cc:1034] running on GCE node
I20240502 01:59:02.585728 16889 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:59:02.586282 16889 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:59:02.587443 16889 hybrid_clock.cc:648] HybridClock initialized: now 1714615142587430 us; error 26 us; skew 500 ppm
I20240502 01:59:02.594748 16889 webserver.cc:460] Webserver started at http://127.7.221.70:33431/ using document root <none> and password file <none>
I20240502 01:59:02.595366 16889 fs_manager.cc:359] Metadata directory not provided
I20240502 01:59:02.595432 16889 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:59:02.599112 16889 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.000s sys 0.003s
I20240502 01:59:02.601066 16904 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:59:02.602532 16889 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:59:02.602706 16889 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "d4e5422f66454511ae5a50ec6ab134ad"
format_stamp: "Formatted at 2024-05-02 01:58:52 on dist-test-slave-dmxt"
I20240502 01:59:02.603112 16889 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:59:02.620296 16889 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:59:02.620899 16889 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:59:02.621053 16889 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:59:02.621578 16889 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:59:02.622416 16889 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:59:02.622467 16889 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:59:02.622529 16889 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:59:02.622555 16889 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:59:02.630863 16889 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.70:34649
I20240502 01:59:02.630909 17021 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.70:34649 every 8 connection(s)
I20240502 01:59:02.632007 16889 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 01:59:02.635149 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 16889
I20240502 01:59:02.635567 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.71:43207
--local_ip_for_outbound_sockets=127.7.221.71
--tserver_master_addrs=127.7.221.126:37149
--webserver_port=43637
--webserver_interface=127.7.221.71
--builtin_ntp_servers=127.7.221.84:36353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:59:02.642060 17022 heartbeater.cc:347] Connected to a master server at 127.7.221.126:37149
I20240502 01:59:02.642313 17022 heartbeater.cc:464] Registering TS with master...
I20240502 01:59:02.643033 17022 heartbeater.cc:511] Master 127.7.221.126:37149 requested a full tablet report, sending...
I20240502 01:59:02.644109 14915 ts_manager.cc:194] Re-registered known tserver with Master: d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649)
I20240502 01:59:02.644814 14915 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.70:42903
W20240502 01:59:02.744740 17025 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:59:02.744935 17025 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:59:02.744982 17025 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:59:02.747890 17025 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:59:02.747951 17025 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:59:02.748028 17025 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.71
I20240502 01:59:02.751511 17025 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.71:43207
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.7.221.71
--webserver_port=43637
--tserver_master_addrs=127.7.221.126:37149
--never_fsync=true
--heap_profile_path=/tmp/kudu.17025
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.71
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:59:02.752418 17025 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:59:02.753513 17025 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:59:02.761804 17033 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:59:02.762611 17031 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:59:02.763216 17030 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:59:02.764024 17025 server_base.cc:1034] running on GCE node
I20240502 01:59:02.764389 17025 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:59:02.764892 17025 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:59:02.766055 17025 hybrid_clock.cc:648] HybridClock initialized: now 1714615142766035 us; error 39 us; skew 500 ppm
I20240502 01:59:02.767920 17025 webserver.cc:460] Webserver started at http://127.7.221.71:43637/ using document root <none> and password file <none>
I20240502 01:59:02.768461 17025 fs_manager.cc:359] Metadata directory not provided
I20240502 01:59:02.768548 17025 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:59:02.772037 17025 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.000s sys 0.003s
I20240502 01:59:02.773852 17039 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:59:02.774847 17025 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.001s
I20240502 01:59:02.774938 17025 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "fc067ee690bb457e8109ad8e4eb43ab8"
format_stamp: "Formatted at 2024-05-02 01:58:53 on dist-test-slave-dmxt"
I20240502 01:59:02.775344 17025 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:59:02.790709 17025 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:59:02.791213 17025 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:59:02.791348 17025 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:59:02.791857 17025 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:59:02.792691 17025 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:59:02.792743 17025 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:59:02.792793 17025 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:59:02.792826 17025 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:59:02.801807 17025 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.71:43207
I20240502 01:59:02.801852 17155 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.71:43207 every 8 connection(s)
I20240502 01:59:02.803007 17025 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 01:59:02.803839 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 17025
I20240502 01:59:02.804320 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.72:37645
--local_ip_for_outbound_sockets=127.7.221.72
--tserver_master_addrs=127.7.221.126:37149
--webserver_port=41171
--webserver_interface=127.7.221.72
--builtin_ntp_servers=127.7.221.84:36353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 01:59:02.813268 17156 heartbeater.cc:347] Connected to a master server at 127.7.221.126:37149
I20240502 01:59:02.813457 17156 heartbeater.cc:464] Registering TS with master...
I20240502 01:59:02.813908 17156 heartbeater.cc:511] Master 127.7.221.126:37149 requested a full tablet report, sending...
I20240502 01:59:02.814869 14915 ts_manager.cc:194] Re-registered known tserver with Master: fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207)
I20240502 01:59:02.815574 14915 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.71:47523
W20240502 01:59:02.910404 17158 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:59:02.910645 17158 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 01:59:02.910691 17158 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:59:02.913489 17158 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 01:59:02.913539 17158 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 01:59:02.913597 17158 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.72
I20240502 01:59:02.917336 17158 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.72:37645
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.7.221.72
--webserver_port=41171
--tserver_master_addrs=127.7.221.126:37149
--never_fsync=true
--heap_profile_path=/tmp/kudu.17158
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.72
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 01:59:02.918112 17158 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 01:59:02.919283 17158 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 01:59:02.927487 17166 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:59:02.928032 17165 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 01:59:02.928777 17168 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 01:59:02.929472 17158 server_base.cc:1034] running on GCE node
I20240502 01:59:02.929852 17158 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 01:59:02.930418 17158 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 01:59:02.931550 17158 hybrid_clock.cc:648] HybridClock initialized: now 1714615142931533 us; error 29 us; skew 500 ppm
I20240502 01:59:02.933310 17158 webserver.cc:460] Webserver started at http://127.7.221.72:41171/ using document root <none> and password file <none>
I20240502 01:59:02.933813 17158 fs_manager.cc:359] Metadata directory not provided
I20240502 01:59:02.933874 17158 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 01:59:02.936975 17158 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:59:02.938638 17174 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 01:59:02.939561 17158 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 01:59:02.939646 17158 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "3b8d615c71e44ff887e0f037c3efcff7"
format_stamp: "Formatted at 2024-05-02 01:58:53 on dist-test-slave-dmxt"
I20240502 01:59:02.939989 17158 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 01:59:02.954360 17158 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 01:59:02.954880 17158 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 01:59:02.955003 17158 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 01:59:02.955411 17158 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 01:59:02.956197 17158 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 01:59:02.956243 17158 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 01:59:02.956279 17158 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 01:59:02.956300 17158 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 01:59:02.963819 17158 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.72:37645
I20240502 01:59:02.963855 17292 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.72:37645 every 8 connection(s)
I20240502 01:59:02.964715 17158 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 01:59:02.970842 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 17158
I20240502 01:59:02.973671 17293 heartbeater.cc:347] Connected to a master server at 127.7.221.126:37149
I20240502 01:59:02.973906 17293 heartbeater.cc:464] Registering TS with master...
I20240502 01:59:02.974516 17293 heartbeater.cc:511] Master 127.7.221.126:37149 requested a full tablet report, sending...
I20240502 01:59:02.975689 14915 ts_manager.cc:194] Re-registered known tserver with Master: 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 01:59:02.976481 14915 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.72:57931
I20240502 01:59:03.350059 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:03.363348 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.379668 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.391475 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.400080 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.407110 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:03.408668 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.415565 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.432548 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.448163 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.449249 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.450708 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:03.453658 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.461551 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.467284 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.469070 16887 heartbeater.cc:503] Master 127.7.221.126:37149 was elected leader, sending a full tablet report...
I20240502 01:59:03.479250 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.491468 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.492679 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.503201 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.504671 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.512305 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:03.519742 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:03.521004 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.535753 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.555204 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.557948 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.564844 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.572782 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.573029 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.583770 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.588450 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.589869 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.592308 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.593394 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.611131 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.614087 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.618994 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.624217 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.631875 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.632267 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.647167 17022 heartbeater.cc:503] Master 127.7.221.126:37149 was elected leader, sending a full tablet report...
I20240502 01:59:03.817741 17156 heartbeater.cc:503] Master 127.7.221.126:37149 was elected leader, sending a full tablet report...
I20240502 01:59:03.848560 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.860889 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.862900 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.898779 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.906803 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:03.911269 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.919915 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.920163 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.922320 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.922946 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:03.925765 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:03.928625 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.929445 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.937628 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.944155 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.952883 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.961201 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.968573 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.969556 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:03.979033 17293 heartbeater.cc:503] Master 127.7.221.126:37149 was elected leader, sending a full tablet report...
I20240502 01:59:03.983371 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:03.983400 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.005856 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.020792 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.046144 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:04.047667 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.056349 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.057610 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.060657 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.061879 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.066442 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:04.073868 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.074018 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.088943 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.090852 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.101928 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.120288 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.121639 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.132020 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.153227 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.161921 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.324739 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:04.327674 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:04.372107 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.377669 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.379710 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.396603 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.397094 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.398766 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.441160 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.448973 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:04.452307 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.452800 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.462627 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.466707 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.469132 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.469835 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.481053 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.481843 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.498996 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.501736 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.511030 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.513513 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.518847 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.521936 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.524204 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.527215 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:04.527357 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.557029 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.569113 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.577781 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.589149 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.594412 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.596544 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.598245 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.613165 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.613675 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.625881 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.641080 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:04.644603 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.650349 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.685842 15331 consensus_queue.cc:237] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:04.688086 15059 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.688482 18058 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.691579 15197 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.692242 16659 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 01:59:04.697295 15266 consensus_peers.cc:485] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e2f4ca025890492880608c7975650969. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:04.701632 16659 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:04.702220 15059 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:04.705024 14908 catalog_manager.cc:5484] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:04.708870 15197 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:04.729518 15197 consensus_queue.cc:237] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:04.729518 15466 consensus_queue.cc:237] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:04.731320 15059 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.731819 15061 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.732328 17910 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.732434 15197 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.732893 17307 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.733196 17307 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.733887 15466 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.734426 17910 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.737116 17910 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:04.738715 15331 consensus_queue.cc:237] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:04.739884 14910 catalog_manager.cc:5484] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:04.740037 18108 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:04.742200 15197 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:04.742949 15195 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 01:59:04.743921 15266 consensus_peers.cc:485] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e2f4ca025890492880608c7975650969. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:04.743850 15465 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:04.745086 18071 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:04.745818 14908 catalog_manager.cc:5484] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:04.746093 15061 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:04.749516 15061 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
W20240502 01:59:04.750089 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 81c8fe21ec7545e79450532b43a61ee7. This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:04.752815 15265 consensus_peers.cc:485] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e2f4ca025890492880608c7975650969. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:04.753360 18071 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:04.753679 15197 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:04.755373 15059 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:04.755735 18084 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:04.756676 15059 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:04.757375 14907 catalog_manager.cc:5484] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:04.762416 15059 consensus_queue.cc:237] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:04.765305 15197 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.765688 18150 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.766389 15331 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 01:59:04.768144 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fcfc8dc4b90a4a6aa7e4035bbfdb4532. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:04.768508 15195 consensus_queue.cc:237] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:04.768764 18150 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.770350 15329 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.770547 15465 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.770740 17471 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.770910 17910 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 01:59:04.771785 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d1c832c721fb4b15bc6d266a1c295401. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:04.775207 17471 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:04.775393 18150 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:04.777273 15465 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
W20240502 01:59:04.777480 15131 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 856f623374824d94913db0cb9faa25d1. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:04.777720 15331 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:04.778120 14906 catalog_manager.cc:5484] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:04.779798 14910 catalog_manager.cc:5484] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:04.785076 15193 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:04.785996 15329 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:04.796945 15193 consensus_queue.cc:237] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:04.798548 15465 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.798949 17910 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 01:59:04.799861 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a901eb3063ea4911a822552db01d365d. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:04.800158 15329 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.801693 17471 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.802006 17471 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:04.805737 15465 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:04.806779 15329 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:04.809650 14914 catalog_manager.cc:5484] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:04.811990 15464 consensus_queue.cc:237] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:04.813576 15059 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.814689 18108 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.814838 15329 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.818697 15464 consensus_queue.cc:237] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:04.819490 18110 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 01:59:04.820506 15396 consensus_peers.cc:485] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 986e7c97e0734e958f364dc5d1167846. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:04.820827 15329 raft_consensus.cc:1261] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.821269 15193 raft_consensus.cc:1261] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.821609 18110 consensus_queue.cc:1035] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.822932 18108 consensus_queue.cc:1035] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 01:59:04.823477 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fc07e430e3e74e3885f263881bd04976. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:04.825441 18110 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:04.825821 15331 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:04.828418 14913 catalog_manager.cc:5484] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:04.833912 18108 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:04.834285 15193 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:04.836768 14908 catalog_manager.cc:5484] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:04.840937 15193 consensus_queue.cc:237] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:04.843508 15061 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.843998 17910 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 01:59:04.844610 15131 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d24986c2ebab46f086e3f1f5a53af26b. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:04.844898 15464 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.846108 15059 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:04.846549 15329 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:04.848546 17910 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.855321 17910 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:04.857439 15061 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:04.857837 14915 catalog_manager.cc:5484] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:04.859617 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.861060 15464 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:04.867640 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:04.874819 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.878852 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.883575 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.892572 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.893128 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.898974 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.910326 15329 consensus_queue.cc:237] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:04.912238 15061 raft_consensus.cc:1261] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.912600 18058 consensus_queue.cc:1035] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.914428 15464 raft_consensus.cc:1261] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.914754 18058 consensus_queue.cc:1035] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 01:59:04.918481 15266 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9c2892024ce942e1bf1834c1b9ad2049. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:04.923592 15465 consensus_queue.cc:237] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:04.924790 18058 raft_consensus.cc:2932] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:04.927595 15193 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 01:59:04.928156 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ac5ec2c673cb4b988d1099e0442ec63f. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:04.928658 15465 raft_consensus.cc:2932] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:04.929805 15329 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.929921 17307 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.930414 18110 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.930680 14908 catalog_manager.cc:5484] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:04.941568 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:04.942806 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.944382 15061 raft_consensus.cc:2932] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:04.952440 18110 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:04.952973 15465 consensus_queue.cc:237] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:04.953459 15329 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:04.953043 15193 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:04.954986 14908 catalog_manager.cc:5484] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:04.955669 15061 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.957960 18108 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:04.960963 15193 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:04.962244 18108 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 01:59:04.962638 15398 consensus_peers.cc:485] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bf73e677bcd049869cadf9f8c9cd3434. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:04.964411 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.971839 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.977255 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:04.984445 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:04.999819 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:05.000072 18279 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.007177 15061 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.008834 14914 catalog_manager.cc:5484] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.014636 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:05.016258 15465 consensus_queue.cc:237] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.019198 15193 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.020761 15329 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.021198 18110 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 01:59:05.022257 15398 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fc07e430e3e74e3885f263881bd04976. This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:05.022413 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fc07e430e3e74e3885f263881bd04976. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.022541 15061 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.022966 18108 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.023922 18108 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.025215 15061 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.026468 15329 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.028519 14908 catalog_manager.cc:5484] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.029217 15061 consensus_queue.cc:237] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.030715 15465 raft_consensus.cc:1261] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 01:59:05.031415 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8f2fb504a49c44edac92fd935555c278. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.031483 15193 raft_consensus.cc:1261] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.031783 18150 consensus_queue.cc:1035] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.032016 18151 consensus_queue.cc:1035] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.035677 18151 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.038957 15465 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.039657 14914 catalog_manager.cc:5484] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.043903 15061 consensus_queue.cc:237] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:05.045612 15329 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.046023 15193 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.048437 15465 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.048590 16653 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.049150 16653 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 01:59:05.050158 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e00d1ed5cde044cb9bdebf787590261a. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.052531 18319 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.054366 15329 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.057171 14914 catalog_manager.cc:5484] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.057979 15465 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.062287 15193 consensus_queue.cc:237] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
W20240502 01:59:05.063910 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 5804597cc507483dba622e6ec44e5ead. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.064644 15465 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.065016 18189 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.067067 15329 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.067744 18189 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.073132 15464 consensus_queue.cc:237] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:05.073323 18189 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
W20240502 01:59:05.075132 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ac5ec2c673cb4b988d1099e0442ec63f. This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:05.075455 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ac5ec2c673cb4b988d1099e0442ec63f. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.075551 15329 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.077409 14908 catalog_manager.cc:5484] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.078714 15464 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.079689 15193 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.080942 15193 consensus_queue.cc:237] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:05.081701 15195 consensus_queue.cc:237] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.083237 15464 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.083263 18279 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 01:59:05.083918 15131 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 856f623374824d94913db0cb9faa25d1. This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:05.084038 15131 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 856f623374824d94913db0cb9faa25d1. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.084632 17910 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.084851 15464 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.085379 15331 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.085467 15330 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.085837 17910 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.085937 17471 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.086153 17910 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 01:59:05.086807 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a901eb3063ea4911a822552db01d365d. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.087080 18279 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.087402 15329 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.087877 17910 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.088263 15330 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.089591 18110 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 01:59:05.089749 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a901eb3063ea4911a822552db01d365d. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.090447 14915 catalog_manager.cc:5484] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.090895 18189 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.091650 15465 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.092790 15464 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.094226 15195 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.097112 14908 catalog_manager.cc:5484] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.099066 15464 consensus_queue.cc:237] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
W20240502 01:59:05.100849 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fcfc8dc4b90a4a6aa7e4035bbfdb4532. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.101251 15061 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.101640 18204 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.103164 15195 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.104013 18110 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.104220 17307 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
W20240502 01:59:05.106690 15399 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fcfc8dc4b90a4a6aa7e4035bbfdb4532. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.106712 15061 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.109215 15195 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.109341 14915 catalog_manager.cc:5484] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.112082 15329 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.114738 14912 catalog_manager.cc:5484] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.116400 15330 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.120829 15195 consensus_queue.cc:237] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:05.123250 15464 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:05.123373 15330 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:05.123821 18109 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:05.124425 17910 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20240502 01:59:05.124924 15131 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 856f623374824d94913db0cb9faa25d1. This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:05.126184 15131 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 856f623374824d94913db0cb9faa25d1. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.127707 18109 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
W20240502 01:59:05.127964 15127 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 856f623374824d94913db0cb9faa25d1. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.129534 15330 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.130311 14915 catalog_manager.cc:5484] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 5 to 6, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.130523 15061 consensus_queue.cc:237] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:05.130438 15464 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.132973 15195 raft_consensus.cc:1261] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.133615 15330 raft_consensus.cc:1261] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.133996 18311 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.136468 15061 consensus_queue.cc:237] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
W20240502 01:59:05.137857 14995 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8ddc239a297a46e48e069b985cb7b350. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.138314 15193 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 01:59:05.139329 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1eb15147ff6c4d0b83b12dec736fc576. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.139606 18311 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.140012 15330 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.140364 18151 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.140650 18311 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.142819 18151 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.144124 18151 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.146217 15193 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.146384 15195 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.146987 15330 consensus_queue.cc:237] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.147450 14915 catalog_manager.cc:5484] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 01:59:05.149003 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 30620f4774b8414eb4e27bd5ecb6ecee. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.149065 15195 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.149401 18071 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.149467 15464 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.150346 18071 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.150728 15330 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.150844 15329 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.152910 14915 catalog_manager.cc:5484] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.162235 15329 consensus_queue.cc:237] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:05.162714 15195 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
W20240502 01:59:05.164561 15265 consensus_peers.cc:485] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e2f4ca025890492880608c7975650969. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.162237 18071 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.165627 15061 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:05.166862 15195 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:05.167305 18084 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20240502 01:59:05.167773 15266 consensus_peers.cc:485] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e2f4ca025890492880608c7975650969. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.167814 14914 catalog_manager.cc:5484] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.169795 16659 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:05.171595 18084 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
W20240502 01:59:05.171908 15266 consensus_peers.cc:485] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e2f4ca025890492880608c7975650969. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.172165 15061 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.174038 15195 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.177003 14908 catalog_manager.cc:5484] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 5 to 6, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.187634 15465 consensus_queue.cc:237] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:05.189196 15061 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.189445 15329 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.189756 17307 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.190061 18204 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 01:59:05.192013 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6d5cffb7fcc54118861463e3b1fa4a8d. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.191970 18204 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.193877 15330 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.194818 14915 catalog_manager.cc:5484] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.207109 15061 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.213479 18446 ts_tablet_manager.cc:902] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:05.214339 18446 tablet_copy_client.cc:308] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:34323
I20240502 01:59:05.214547 15061 consensus_queue.cc:237] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
W20240502 01:59:05.216357 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e00d1ed5cde044cb9bdebf787590261a. This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:05.216737 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e00d1ed5cde044cb9bdebf787590261a. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.217514 15465 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.217514 15330 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.217933 18311 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.218951 18311 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.219483 18456 ts_tablet_manager.cc:902] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:05.219897 15488 tablet_copy_service.cc:140] P 7b66a8940fe54e21bec1b2db25b2fed5: Received BeginTabletCopySession request for tablet 986e7c97e0734e958f364dc5d1167846 from peer fc067ee690bb457e8109ad8e4eb43ab8 ({username='slave'} at 127.7.221.71:50737)
I20240502 01:59:05.220041 15488 tablet_copy_service.cc:161] P 7b66a8940fe54e21bec1b2db25b2fed5: Beginning new tablet copy session on tablet 986e7c97e0734e958f364dc5d1167846 from peer fc067ee690bb457e8109ad8e4eb43ab8 at {username='slave'} at 127.7.221.71:50737: session id = fc067ee690bb457e8109ad8e4eb43ab8-986e7c97e0734e958f364dc5d1167846
I20240502 01:59:05.220213 18311 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.221462 15488 tablet_copy_source_session.cc:205] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.222923 15464 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.225744 18446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 986e7c97e0734e958f364dc5d1167846. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.226785 15330 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.229169 14915 catalog_manager.cc:5484] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.229712 15464 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.232405 18460 ts_tablet_manager.cc:902] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:05.233381 18460 tablet_copy_client.cc:308] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:05.238250 15217 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet 81c8fe21ec7545e79450532b43a61ee7 from peer 3b8d615c71e44ff887e0f037c3efcff7 ({username='slave'} at 127.7.221.72:57673)
I20240502 01:59:05.238410 15217 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet 81c8fe21ec7545e79450532b43a61ee7 from peer 3b8d615c71e44ff887e0f037c3efcff7 at {username='slave'} at 127.7.221.72:57673: session id = 3b8d615c71e44ff887e0f037c3efcff7-81c8fe21ec7545e79450532b43a61ee7
I20240502 01:59:05.239802 18456 tablet_copy_client.cc:308] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:46123
I20240502 01:59:05.240463 15217 tablet_copy_source_session.cc:205] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.246109 18446 tablet_copy_client.cc:783] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.246331 18446 tablet_copy_client.cc:647] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.247272 15195 consensus_queue.cc:237] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.248885 18446 tablet_copy_client.cc:523] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.249665 15081 tablet_copy_service.cc:140] P ddae2a636423489693c12b7d7d166c09: Received BeginTabletCopySession request for tablet d1c832c721fb4b15bc6d266a1c295401 from peer d4e5422f66454511ae5a50ec6ab134ad ({username='slave'} at 127.7.221.70:34963)
I20240502 01:59:05.249826 15081 tablet_copy_service.cc:161] P ddae2a636423489693c12b7d7d166c09: Beginning new tablet copy session on tablet d1c832c721fb4b15bc6d266a1c295401 from peer d4e5422f66454511ae5a50ec6ab134ad at {username='slave'} at 127.7.221.70:34963: session id = d4e5422f66454511ae5a50ec6ab134ad-d1c832c721fb4b15bc6d266a1c295401
I20240502 01:59:05.250938 18446 tablet_bootstrap.cc:492] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap starting.
I20240502 01:59:05.251719 15081 tablet_copy_source_session.cc:205] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.252985 15464 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.253741 17471 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.255369 18456 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d1c832c721fb4b15bc6d266a1c295401. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.256843 15061 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 01:59:05.256989 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 81c8fe21ec7545e79450532b43a61ee7. This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:05.257133 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 81c8fe21ec7545e79450532b43a61ee7. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.257469 17471 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.257900 18446 log.cc:824] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8: Log is configured to *not* fsync() on all Append() calls
I20240502 01:59:05.258242 17471 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.258419 18460 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81c8fe21ec7545e79450532b43a61ee7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.261198 14909 catalog_manager.cc:5484] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.265301 15464 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.269425 18460 tablet_copy_client.cc:783] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.269645 18460 tablet_copy_client.cc:647] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.271318 18460 tablet_copy_client.cc:523] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.273222 18460 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap starting.
I20240502 01:59:05.275897 15059 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.277001 15061 consensus_queue.cc:237] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
W20240502 01:59:05.279111 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d1c832c721fb4b15bc6d266a1c295401. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.279965 15330 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.280711 18319 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.281561 18460 log.cc:824] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Log is configured to *not* fsync() on all Append() calls
I20240502 01:59:05.282603 15195 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.283157 18319 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.285576 18150 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.288081 18456 tablet_copy_client.cc:783] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.289145 14909 catalog_manager.cc:5484] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.289695 15330 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.289853 18456 tablet_copy_client.cc:647] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.295804 15195 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.297080 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
W20240502 01:59:05.299273 14995 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d1c832c721fb4b15bc6d266a1c295401. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.301199 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:05.305807 18456 tablet_copy_client.cc:523] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.307251 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:05.308405 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:05.308653 18456 tablet_bootstrap.cc:492] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap starting.
I20240502 01:59:05.312945 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:05.315989 18456 log.cc:824] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad: Log is configured to *not* fsync() on all Append() calls
I20240502 01:59:05.323367 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:05.343001 18517 ts_tablet_manager.cc:902] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:05.346808 18456 tablet_bootstrap.cc:492] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:05.347304 18456 tablet_bootstrap.cc:492] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap complete.
I20240502 01:59:05.347689 18456 ts_tablet_manager.cc:1367] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad: Time spent bootstrapping tablet: real 0.039s user 0.032s sys 0.002s
I20240502 01:59:05.350224 18456 raft_consensus.cc:348] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.350600 18456 raft_consensus.cc:729] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d4e5422f66454511ae5a50ec6ab134ad, State: Initialized, Role: LEARNER
I20240502 01:59:05.351012 18456 consensus_queue.cc:260] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.351982 18456 ts_tablet_manager.cc:1398] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad: Time spent starting tablet: real 0.004s user 0.000s sys 0.003s
I20240502 01:59:05.353724 15081 tablet_copy_service.cc:342] P ddae2a636423489693c12b7d7d166c09: Request end of tablet copy session d4e5422f66454511ae5a50ec6ab134ad-d1c832c721fb4b15bc6d266a1c295401 received from {username='slave'} at 127.7.221.70:34963
I20240502 01:59:05.353830 15081 tablet_copy_service.cc:433] P ddae2a636423489693c12b7d7d166c09: ending tablet copy session d4e5422f66454511ae5a50ec6ab134ad-d1c832c721fb4b15bc6d266a1c295401 on tablet d1c832c721fb4b15bc6d266a1c295401 with peer d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:05.354750 18517 tablet_copy_client.cc:308] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:05.376947 18460 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:05.377394 18460 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap complete.
I20240502 01:59:05.377774 18460 ts_tablet_manager.cc:1367] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Time spent bootstrapping tablet: real 0.105s user 0.034s sys 0.000s
I20240502 01:59:05.381425 18460 raft_consensus.cc:348] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:05.381803 18460 raft_consensus.cc:729] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b8d615c71e44ff887e0f037c3efcff7, State: Initialized, Role: LEARNER
I20240502 01:59:05.382711 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:05.383144 18460 consensus_queue.cc:260] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:05.383855 15061 consensus_queue.cc:237] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
W20240502 01:59:05.386833 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1eb15147ff6c4d0b83b12dec736fc576. This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:05.387159 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1eb15147ff6c4d0b83b12dec736fc576. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.387671 15330 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.388600 18386 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.389222 18446 tablet_bootstrap.cc:492] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:05.389609 18446 tablet_bootstrap.cc:492] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap complete.
I20240502 01:59:05.389928 18446 ts_tablet_manager.cc:1367] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8: Time spent bootstrapping tablet: real 0.139s user 0.033s sys 0.000s
I20240502 01:59:05.390733 18386 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.393419 18446 raft_consensus.cc:348] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:05.393698 18446 raft_consensus.cc:729] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fc067ee690bb457e8109ad8e4eb43ab8, State: Initialized, Role: LEARNER
I20240502 01:59:05.393757 15195 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.394071 18446 consensus_queue.cc:260] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:05.394302 18150 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.394997 15330 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.395404 18446 ts_tablet_manager.cc:1398] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8: Time spent starting tablet: real 0.005s user 0.006s sys 0.000s
I20240502 01:59:05.397739 14915 catalog_manager.cc:5484] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.398500 15195 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.398766 15488 tablet_copy_service.cc:342] P 7b66a8940fe54e21bec1b2db25b2fed5: Request end of tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-986e7c97e0734e958f364dc5d1167846 received from {username='slave'} at 127.7.221.71:50737
I20240502 01:59:05.398849 15488 tablet_copy_service.cc:433] P 7b66a8940fe54e21bec1b2db25b2fed5: ending tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-986e7c97e0734e958f364dc5d1167846 on tablet 986e7c97e0734e958f364dc5d1167846 with peer fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:05.400872 15464 consensus_queue.cc:237] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:05.402351 15330 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 01:59:05.402902 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6d5cffb7fcc54118861463e3b1fa4a8d. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.403576 18110 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 01:59:05.404707 15399 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6d5cffb7fcc54118861463e3b1fa4a8d. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.405014 15061 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.406183 18279 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.409474 18279 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.410869 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:05.411152 15059 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.412369 15330 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.413573 14908 catalog_manager.cc:5484] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.417382 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:05.419811 15217 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet d24986c2ebab46f086e3f1f5a53af26b from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:43177)
I20240502 01:59:05.419899 15217 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet d24986c2ebab46f086e3f1f5a53af26b from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:43177: session id = 84987ad9e4a54d96af8569e4aaf0ed64-d24986c2ebab46f086e3f1f5a53af26b
I20240502 01:59:05.420760 15217 tablet_copy_source_session.cc:205] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.422163 18456 ts_tablet_manager.cc:902] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:05.422570 18456 tablet_copy_client.cc:308] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:46123
I20240502 01:59:05.425180 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:05.425318 15081 tablet_copy_service.cc:140] P ddae2a636423489693c12b7d7d166c09: Received BeginTabletCopySession request for tablet 8f2fb504a49c44edac92fd935555c278 from peer d4e5422f66454511ae5a50ec6ab134ad ({username='slave'} at 127.7.221.70:34963)
I20240502 01:59:05.425441 15081 tablet_copy_service.cc:161] P ddae2a636423489693c12b7d7d166c09: Beginning new tablet copy session on tablet 8f2fb504a49c44edac92fd935555c278 from peer d4e5422f66454511ae5a50ec6ab134ad at {username='slave'} at 127.7.221.70:34963: session id = d4e5422f66454511ae5a50ec6ab134ad-8f2fb504a49c44edac92fd935555c278
I20240502 01:59:05.426432 15081 tablet_copy_source_session.cc:205] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.428184 18456 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8f2fb504a49c44edac92fd935555c278. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.429186 18460 ts_tablet_manager.cc:1398] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Time spent starting tablet: real 0.051s user 0.000s sys 0.004s
I20240502 01:59:05.430070 18456 tablet_copy_client.cc:783] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.430217 18456 tablet_copy_client.cc:647] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.430794 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:05.436868 18456 tablet_copy_client.cc:523] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.436578 18517 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d24986c2ebab46f086e3f1f5a53af26b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.439378 18456 tablet_bootstrap.cc:492] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap starting.
I20240502 01:59:05.441190 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:05.450980 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:05.453004 15217 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-81c8fe21ec7545e79450532b43a61ee7 received from {username='slave'} at 127.7.221.72:57673
I20240502 01:59:05.453100 15217 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-81c8fe21ec7545e79450532b43a61ee7 on tablet 81c8fe21ec7545e79450532b43a61ee7 with peer 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:05.467059 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:05.467809 18456 tablet_bootstrap.cc:492] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:05.468077 18456 tablet_bootstrap.cc:492] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap complete.
I20240502 01:59:05.468245 18456 ts_tablet_manager.cc:1367] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad: Time spent bootstrapping tablet: real 0.030s user 0.022s sys 0.001s
I20240502 01:59:05.468815 18456 raft_consensus.cc:348] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.468946 18456 raft_consensus.cc:729] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d4e5422f66454511ae5a50ec6ab134ad, State: Initialized, Role: LEARNER
I20240502 01:59:05.469051 18456 consensus_queue.cc:260] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.469789 18456 ts_tablet_manager.cc:1398] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:05.470821 18517 tablet_copy_client.cc:783] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.471467 15081 tablet_copy_service.cc:342] P ddae2a636423489693c12b7d7d166c09: Request end of tablet copy session d4e5422f66454511ae5a50ec6ab134ad-8f2fb504a49c44edac92fd935555c278 received from {username='slave'} at 127.7.221.70:34963
I20240502 01:59:05.471551 15081 tablet_copy_service.cc:433] P ddae2a636423489693c12b7d7d166c09: ending tablet copy session d4e5422f66454511ae5a50ec6ab134ad-8f2fb504a49c44edac92fd935555c278 on tablet 8f2fb504a49c44edac92fd935555c278 with peer d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:05.482237 18460 ts_tablet_manager.cc:902] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:05.482798 18460 tablet_copy_client.cc:308] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:05.483179 15217 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet a901eb3063ea4911a822552db01d365d from peer 3b8d615c71e44ff887e0f037c3efcff7 ({username='slave'} at 127.7.221.72:57673)
I20240502 01:59:05.483276 15217 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet a901eb3063ea4911a822552db01d365d from peer 3b8d615c71e44ff887e0f037c3efcff7 at {username='slave'} at 127.7.221.72:57673: session id = 3b8d615c71e44ff887e0f037c3efcff7-a901eb3063ea4911a822552db01d365d
I20240502 01:59:05.483670 15195 consensus_queue.cc:237] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:05.484203 15217 tablet_copy_source_session.cc:205] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.485502 15059 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.485689 15464 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.485899 17910 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 01:59:05.486058 15127 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d24986c2ebab46f086e3f1f5a53af26b. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.486881 17910 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.488281 17910 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
W20240502 01:59:05.491312 15131 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.491796 15059 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.494854 14913 catalog_manager.cc:5484] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.495291 15464 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.495891 18517 tablet_copy_client.cc:647] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.480063 18557 ts_tablet_manager.cc:902] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:05.497501 18456 ts_tablet_manager.cc:902] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:05.497915 18456 tablet_copy_client.cc:308] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:05.498060 18557 tablet_copy_client.cc:308] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 01:59:05.499120 18460 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a901eb3063ea4911a822552db01d365d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.502002 18460 tablet_copy_client.cc:783] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.502161 18460 tablet_copy_client.cc:647] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.502732 15217 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet a901eb3063ea4911a822552db01d365d from peer d4e5422f66454511ae5a50ec6ab134ad ({username='slave'} at 127.7.221.70:59193)
I20240502 01:59:05.502853 15217 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet a901eb3063ea4911a822552db01d365d from peer d4e5422f66454511ae5a50ec6ab134ad at {username='slave'} at 127.7.221.70:59193: session id = d4e5422f66454511ae5a50ec6ab134ad-a901eb3063ea4911a822552db01d365d
I20240502 01:59:05.503715 15217 tablet_copy_source_session.cc:205] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.504873 18456 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a901eb3063ea4911a822552db01d365d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.507244 18456 tablet_copy_client.cc:783] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.507393 18456 tablet_copy_client.cc:647] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.508147 18460 tablet_copy_client.cc:523] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.509382 18460 tablet_bootstrap.cc:492] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap starting.
I20240502 01:59:05.509603 18517 tablet_copy_client.cc:523] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.511696 18517 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
I20240502 01:59:05.512168 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet 9c2892024ce942e1bf1834c1b9ad2049 from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:51343)
I20240502 01:59:05.512315 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet 9c2892024ce942e1bf1834c1b9ad2049 from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:51343: session id = 84987ad9e4a54d96af8569e4aaf0ed64-9c2892024ce942e1bf1834c1b9ad2049
I20240502 01:59:05.513774 15351 tablet_copy_source_session.cc:205] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.514909 18557 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9c2892024ce942e1bf1834c1b9ad2049. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.515394 18446 ts_tablet_manager.cc:902] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:05.515847 18446 tablet_copy_client.cc:308] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:34323
I20240502 01:59:05.516779 15488 tablet_copy_service.cc:140] P 7b66a8940fe54e21bec1b2db25b2fed5: Received BeginTabletCopySession request for tablet ac5ec2c673cb4b988d1099e0442ec63f from peer fc067ee690bb457e8109ad8e4eb43ab8 ({username='slave'} at 127.7.221.71:50737)
I20240502 01:59:05.516896 15488 tablet_copy_service.cc:161] P 7b66a8940fe54e21bec1b2db25b2fed5: Beginning new tablet copy session on tablet ac5ec2c673cb4b988d1099e0442ec63f from peer fc067ee690bb457e8109ad8e4eb43ab8 at {username='slave'} at 127.7.221.71:50737: session id = fc067ee690bb457e8109ad8e4eb43ab8-ac5ec2c673cb4b988d1099e0442ec63f
I20240502 01:59:05.517287 18557 tablet_copy_client.cc:783] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.517419 18557 tablet_copy_client.cc:647] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.517866 15488 tablet_copy_source_session.cc:205] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.518739 18446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ac5ec2c673cb4b988d1099e0442ec63f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.520783 18446 tablet_copy_client.cc:783] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.520920 18446 tablet_copy_client.cc:647] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.523978 18557 tablet_copy_client.cc:523] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.525384 18557 tablet_bootstrap.cc:492] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
I20240502 01:59:05.526088 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:05.527678 15059 consensus_queue.cc:237] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:05.528682 18590 ts_tablet_manager.cc:902] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:05.529271 18590 tablet_copy_client.cc:308] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:34323
I20240502 01:59:05.530048 15464 raft_consensus.cc:1261] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 01:59:05.530117 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 53edbd87d0d5471ca7626ca2071f75eb. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.530606 15330 raft_consensus.cc:1261] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.530827 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:05.531113 18311 consensus_queue.cc:1035] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.532244 18311 consensus_queue.cc:1035] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.533995 15488 tablet_copy_service.cc:140] P 7b66a8940fe54e21bec1b2db25b2fed5: Received BeginTabletCopySession request for tablet ac5ec2c673cb4b988d1099e0442ec63f from peer 3b8d615c71e44ff887e0f037c3efcff7 ({username='slave'} at 127.7.221.72:51521)
I20240502 01:59:05.534101 15488 tablet_copy_service.cc:161] P 7b66a8940fe54e21bec1b2db25b2fed5: Beginning new tablet copy session on tablet ac5ec2c673cb4b988d1099e0442ec63f from peer 3b8d615c71e44ff887e0f037c3efcff7 at {username='slave'} at 127.7.221.72:51521: session id = 3b8d615c71e44ff887e0f037c3efcff7-ac5ec2c673cb4b988d1099e0442ec63f
I20240502 01:59:05.535064 15488 tablet_copy_source_session.cc:205] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.538193 15059 consensus_queue.cc:237] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
W20240502 01:59:05.540184 14995 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d1c832c721fb4b15bc6d266a1c295401. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.540906 16974 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
W20240502 01:59:05.540989 14995 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d1c832c721fb4b15bc6d266a1c295401. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.541527 15195 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:05.542212 15329 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:05.542359 18584 ts_tablet_manager.cc:902] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:05.543102 18584 tablet_copy_client.cc:308] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:34323
I20240502 01:59:05.543191 18311 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.544390 18150 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:05.544670 18150 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:05.545851 16974 raft_consensus.cc:858] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }.
Pending config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }:
New config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
W20240502 01:59:05.545938 16974 raft_consensus.cc:1565] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Could not prepare op 'term: 1 index: 6' and following 0 ops. Status for this op: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }.
Pending config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }:
New config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:05.546846 16974 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.547987 18150 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.551836 15329 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.555052 18456 tablet_copy_client.cc:523] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.555289 14908 catalog_manager.cc:5484] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 5 to 6, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.555610 15195 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.556437 18456 tablet_bootstrap.cc:492] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap starting.
I20240502 01:59:05.558233 15488 tablet_copy_service.cc:140] P 7b66a8940fe54e21bec1b2db25b2fed5: Received BeginTabletCopySession request for tablet bf73e677bcd049869cadf9f8c9cd3434 from peer d4e5422f66454511ae5a50ec6ab134ad ({username='slave'} at 127.7.221.70:44607)
I20240502 01:59:05.558328 15488 tablet_copy_service.cc:161] P 7b66a8940fe54e21bec1b2db25b2fed5: Beginning new tablet copy session on tablet bf73e677bcd049869cadf9f8c9cd3434 from peer d4e5422f66454511ae5a50ec6ab134ad at {username='slave'} at 127.7.221.70:44607: session id = d4e5422f66454511ae5a50ec6ab134ad-bf73e677bcd049869cadf9f8c9cd3434
I20240502 01:59:05.559289 15488 tablet_copy_source_session.cc:205] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.561692 18517 log.cc:824] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64: Log is configured to *not* fsync() on all Append() calls
I20240502 01:59:05.563165 18584 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bf73e677bcd049869cadf9f8c9cd3434. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.568768 18446 tablet_copy_client.cc:523] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.570281 18446 tablet_bootstrap.cc:492] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap starting.
I20240502 01:59:05.579244 18596 ts_tablet_manager.cc:902] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:05.579819 18596 tablet_copy_client.cc:308] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:05.580345 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet 856f623374824d94913db0cb9faa25d1 from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:43177)
I20240502 01:59:05.580469 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet 856f623374824d94913db0cb9faa25d1 from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:43177: session id = 84987ad9e4a54d96af8569e4aaf0ed64-856f623374824d94913db0cb9faa25d1
I20240502 01:59:05.581485 15216 tablet_copy_source_session.cc:205] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.582624 18311 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.585402 14908 catalog_manager.cc:5484] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.591223 18151 raft_consensus.cc:1050] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09: attempting to promote NON_VOTER d4e5422f66454511ae5a50ec6ab134ad to VOTER
I20240502 01:59:05.591356 18584 tablet_copy_client.cc:783] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.591543 18584 tablet_copy_client.cc:647] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.591800 18151 consensus_queue.cc:237] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:05.592423 18614 ts_tablet_manager.cc:902] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:05.592985 18614 tablet_copy_client.cc:308] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:34323
W20240502 01:59:05.596108 14995 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d1c832c721fb4b15bc6d266a1c295401. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.596846 15465 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
W20240502 01:59:05.598796 14995 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d1c832c721fb4b15bc6d266a1c295401. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.598896 16974 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.600411 15330 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:05.600859 15195 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:05.601500 18584 tablet_copy_client.cc:523] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.601917 15488 tablet_copy_service.cc:140] P 7b66a8940fe54e21bec1b2db25b2fed5: Received BeginTabletCopySession request for tablet fc07e430e3e74e3885f263881bd04976 from peer 3b8d615c71e44ff887e0f037c3efcff7 ({username='slave'} at 127.7.221.72:51521)
I20240502 01:59:05.602006 15488 tablet_copy_service.cc:161] P 7b66a8940fe54e21bec1b2db25b2fed5: Beginning new tablet copy session on tablet fc07e430e3e74e3885f263881bd04976 from peer 3b8d615c71e44ff887e0f037c3efcff7 at {username='slave'} at 127.7.221.72:51521: session id = 3b8d615c71e44ff887e0f037c3efcff7-fc07e430e3e74e3885f263881bd04976
I20240502 01:59:05.603016 18584 tablet_bootstrap.cc:492] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap starting.
I20240502 01:59:05.603088 15488 tablet_copy_source_session.cc:205] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.604061 18319 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:05.604024 15330 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.605213 18319 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:05.605726 16973 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:05.607554 18319 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:05.608834 18319 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.612066 15330 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.612583 15195 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.614363 18614 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc07e430e3e74e3885f263881bd04976. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.615156 14908 catalog_manager.cc:5484] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.618158 15059 consensus_queue.cc:237] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:05.619735 15195 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 01:59:05.619762 14995 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d1c832c721fb4b15bc6d266a1c295401. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.620147 18311 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:05.620494 15330 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:05.622614 18150 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:05.624748 18311 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.626231 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d1c832c721fb4b15bc6d266a1c295401 with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:05.626657 18614 tablet_copy_client.cc:783] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.626876 18614 tablet_copy_client.cc:647] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.627825 14915 catalog_manager.cc:5484] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.628592 15195 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.633951 15330 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.634131 17108 raft_consensus.cc:1203] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 01:59:05.610652 16973 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.644660 16974 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:05.645282 18517 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:05.645237 18319 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:05.645663 18517 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:05.645978 18517 ts_tablet_manager.cc:1367] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.134s user 0.029s sys 0.000s
I20240502 01:59:05.646350 16974 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.647123 18634 ts_tablet_manager.cc:902] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:05.647588 18634 tablet_copy_client.cc:308] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:34323
I20240502 01:59:05.648540 15488 tablet_copy_service.cc:140] P 7b66a8940fe54e21bec1b2db25b2fed5: Received BeginTabletCopySession request for tablet fc07e430e3e74e3885f263881bd04976 from peer d4e5422f66454511ae5a50ec6ab134ad ({username='slave'} at 127.7.221.70:44607)
I20240502 01:59:05.648640 15488 tablet_copy_service.cc:161] P 7b66a8940fe54e21bec1b2db25b2fed5: Beginning new tablet copy session on tablet fc07e430e3e74e3885f263881bd04976 from peer d4e5422f66454511ae5a50ec6ab134ad at {username='slave'} at 127.7.221.70:44607: session id = d4e5422f66454511ae5a50ec6ab134ad-fc07e430e3e74e3885f263881bd04976
I20240502 01:59:05.648597 18517 raft_consensus.cc:348] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:05.648844 18517 raft_consensus.cc:729] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:05.649190 18517 consensus_queue.cc:260] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:05.650064 18517 ts_tablet_manager.cc:1398] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.004s user 0.004s sys 0.000s
I20240502 01:59:05.650810 15330 consensus_queue.cc:237] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.651014 15488 tablet_copy_source_session.cc:205] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5: Tablet Copy: opened 0 blocks and 1 log segments
W20240502 01:59:05.652298 15265 consensus_peers.cc:485] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 091d27d51acc4db0a1c7d20deeff5c10. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.652837 15059 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.653215 18071 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.654955 18634 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc07e430e3e74e3885f263881bd04976. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.656517 18557 tablet_bootstrap.cc:492] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:05.656647 18614 tablet_copy_client.cc:523] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.656802 18557 tablet_bootstrap.cc:492] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:05.656850 18634 tablet_copy_client.cc:783] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.656989 18557 ts_tablet_manager.cc:1367] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.132s user 0.024s sys 0.006s
I20240502 01:59:05.656996 18634 tablet_copy_client.cc:647] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.657689 15465 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.657727 18557 raft_consensus.cc:348] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:05.657828 18557 raft_consensus.cc:729] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:05.657909 18557 consensus_queue.cc:260] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:05.658061 18071 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.658174 18557 ts_tablet_manager.cc:1398] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 01:59:05.659639 18614 tablet_bootstrap.cc:492] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap starting.
I20240502 01:59:05.660732 18596 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 856f623374824d94913db0cb9faa25d1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.661403 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-d24986c2ebab46f086e3f1f5a53af26b received from {username='slave'} at 127.7.221.69:43177
I20240502 01:59:05.661475 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-d24986c2ebab46f086e3f1f5a53af26b on tablet d24986c2ebab46f086e3f1f5a53af26b with peer 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:05.662688 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-9c2892024ce942e1bf1834c1b9ad2049 received from {username='slave'} at 127.7.221.69:51343
I20240502 01:59:05.662779 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-9c2892024ce942e1bf1834c1b9ad2049 on tablet 9c2892024ce942e1bf1834c1b9ad2049 with peer 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:05.663738 18557 ts_tablet_manager.cc:902] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:05.664057 18557 tablet_copy_client.cc:308] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:46123
I20240502 01:59:05.665875 18596 tablet_copy_client.cc:783] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.666093 18596 tablet_copy_client.cc:647] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.667219 18596 tablet_copy_client.cc:523] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.668329 18590 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ac5ec2c673cb4b988d1099e0442ec63f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.669131 18634 tablet_copy_client.cc:523] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.669785 15081 tablet_copy_service.cc:140] P ddae2a636423489693c12b7d7d166c09: Received BeginTabletCopySession request for tablet 8ddc239a297a46e48e069b985cb7b350 from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:51825)
I20240502 01:59:05.669874 15081 tablet_copy_service.cc:161] P ddae2a636423489693c12b7d7d166c09: Beginning new tablet copy session on tablet 8ddc239a297a46e48e069b985cb7b350 from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:51825: session id = 84987ad9e4a54d96af8569e4aaf0ed64-8ddc239a297a46e48e069b985cb7b350
I20240502 01:59:05.670241 18634 tablet_bootstrap.cc:492] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap starting.
I20240502 01:59:05.671290 15081 tablet_copy_source_session.cc:205] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.672036 18557 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8ddc239a297a46e48e069b985cb7b350. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.673794 18557 tablet_copy_client.cc:783] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.673923 18557 tablet_copy_client.cc:647] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.675173 18557 tablet_copy_client.cc:523] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.676164 18557 tablet_bootstrap.cc:492] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
I20240502 01:59:05.676617 17224 tablet_service.cc:1514] Processing DeleteTablet for tablet d1c832c721fb4b15bc6d266a1c295401 with delete_type TABLET_DATA_TOMBSTONED (TS 3b8d615c71e44ff887e0f037c3efcff7 not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:37116
I20240502 01:59:05.678779 18646 ts_tablet_manager.cc:902] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:05.679330 18646 tablet_copy_client.cc:308] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:34323
I20240502 01:59:05.679836 15488 tablet_copy_service.cc:140] P 7b66a8940fe54e21bec1b2db25b2fed5: Received BeginTabletCopySession request for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 from peer fc067ee690bb457e8109ad8e4eb43ab8 ({username='slave'} at 127.7.221.71:50737)
I20240502 01:59:05.679937 15488 tablet_copy_service.cc:161] P 7b66a8940fe54e21bec1b2db25b2fed5: Beginning new tablet copy session on tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 from peer fc067ee690bb457e8109ad8e4eb43ab8 at {username='slave'} at 127.7.221.71:50737: session id = fc067ee690bb457e8109ad8e4eb43ab8-fcfc8dc4b90a4a6aa7e4035bbfdb4532
I20240502 01:59:05.680837 15488 tablet_copy_source_session.cc:205] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.681416 18590 tablet_copy_client.cc:783] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.681332 18071 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.681545 18590 tablet_copy_client.cc:647] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.682499 18590 tablet_copy_client.cc:523] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.683528 15465 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.683952 18590 tablet_bootstrap.cc:492] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap starting.
I20240502 01:59:05.685649 14908 catalog_manager.cc:5484] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.686143 18610 ts_tablet_manager.cc:902] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:05.686791 18610 tablet_copy_client.cc:308] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 01:59:05.688712 15059 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.691175 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet e2f4ca025890492880608c7975650969 from peer d4e5422f66454511ae5a50ec6ab134ad ({username='slave'} at 127.7.221.70:33955)
I20240502 01:59:05.691280 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet e2f4ca025890492880608c7975650969 from peer d4e5422f66454511ae5a50ec6ab134ad at {username='slave'} at 127.7.221.70:33955: session id = d4e5422f66454511ae5a50ec6ab134ad-e2f4ca025890492880608c7975650969
I20240502 01:59:05.692195 15351 tablet_copy_source_session.cc:205] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.697549 18676 ts_tablet_manager.cc:902] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:05.698100 18676 tablet_copy_client.cc:308] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 01:59:05.698539 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet 30620f4774b8414eb4e27bd5ecb6ecee from peer d4e5422f66454511ae5a50ec6ab134ad ({username='slave'} at 127.7.221.70:33955)
I20240502 01:59:05.698649 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet 30620f4774b8414eb4e27bd5ecb6ecee from peer d4e5422f66454511ae5a50ec6ab134ad at {username='slave'} at 127.7.221.70:33955: session id = d4e5422f66454511ae5a50ec6ab134ad-30620f4774b8414eb4e27bd5ecb6ecee
I20240502 01:59:05.699429 15351 tablet_copy_source_session.cc:205] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 0 blocks and 1 log segments
W20240502 01:59:05.700004 14902 catalog_manager.cc:4797] TS 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): delete failed for tablet d1c832c721fb4b15bc6d266a1c295401 because the tablet was not found. No further retry: Not found: Tablet not found: d1c832c721fb4b15bc6d266a1c295401
I20240502 01:59:05.700841 18676 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 30620f4774b8414eb4e27bd5ecb6ecee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.703562 18676 tablet_copy_client.cc:783] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.703785 18676 tablet_copy_client.cc:647] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.706648 18637 ts_tablet_manager.cc:902] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:05.707191 18654 ts_tablet_manager.cc:902] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:05.707463 18637 tablet_copy_client.cc:308] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:46123
I20240502 01:59:05.707758 18654 tablet_copy_client.cc:308] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:05.708688 18646 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.710955 18646 tablet_copy_client.cc:783] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.711128 18646 tablet_copy_client.cc:647] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.711233 15081 tablet_copy_service.cc:140] P ddae2a636423489693c12b7d7d166c09: Received BeginTabletCopySession request for tablet e00d1ed5cde044cb9bdebf787590261a from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:51825)
I20240502 01:59:05.711336 15081 tablet_copy_service.cc:161] P ddae2a636423489693c12b7d7d166c09: Beginning new tablet copy session on tablet e00d1ed5cde044cb9bdebf787590261a from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:51825: session id = 84987ad9e4a54d96af8569e4aaf0ed64-e00d1ed5cde044cb9bdebf787590261a
I20240502 01:59:05.712363 15081 tablet_copy_source_session.cc:205] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.714179 18610 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e2f4ca025890492880608c7975650969. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.715984 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet 856f623374824d94913db0cb9faa25d1 from peer fc067ee690bb457e8109ad8e4eb43ab8 ({username='slave'} at 127.7.221.71:39719)
I20240502 01:59:05.716121 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet 856f623374824d94913db0cb9faa25d1 from peer fc067ee690bb457e8109ad8e4eb43ab8 at {username='slave'} at 127.7.221.71:39719: session id = fc067ee690bb457e8109ad8e4eb43ab8-856f623374824d94913db0cb9faa25d1
I20240502 01:59:05.717067 15216 tablet_copy_source_session.cc:205] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.717425 18614 tablet_bootstrap.cc:492] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:05.717648 18517 ts_tablet_manager.cc:902] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:05.717748 18610 tablet_copy_client.cc:783] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.717967 18610 tablet_copy_client.cc:647] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.718145 18517 tablet_copy_client.cc:308] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:34323
I20240502 01:59:05.719141 18614 tablet_bootstrap.cc:492] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap complete.
I20240502 01:59:05.719323 18614 ts_tablet_manager.cc:1367] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7: Time spent bootstrapping tablet: real 0.060s user 0.028s sys 0.000s
I20240502 01:59:05.719906 18614 raft_consensus.cc:348] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.720039 18614 raft_consensus.cc:729] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b8d615c71e44ff887e0f037c3efcff7, State: Initialized, Role: LEARNER
I20240502 01:59:05.720131 18614 consensus_queue.cc:260] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.721050 18637 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e00d1ed5cde044cb9bdebf787590261a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.723021 18614 ts_tablet_manager.cc:1398] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:59:05.723155 15195 consensus_queue.cc:237] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:05.723819 15488 tablet_copy_service.cc:342] P 7b66a8940fe54e21bec1b2db25b2fed5: Request end of tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-fc07e430e3e74e3885f263881bd04976 received from {username='slave'} at 127.7.221.72:51521
I20240502 01:59:05.723902 15488 tablet_copy_service.cc:433] P 7b66a8940fe54e21bec1b2db25b2fed5: ending tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-fc07e430e3e74e3885f263881bd04976 on tablet fc07e430e3e74e3885f263881bd04976 with peer 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:05.723920 18634 tablet_bootstrap.cc:492] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:05.724197 18634 tablet_bootstrap.cc:492] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap complete.
I20240502 01:59:05.724423 18634 ts_tablet_manager.cc:1367] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: Time spent bootstrapping tablet: real 0.054s user 0.022s sys 0.004s
I20240502 01:59:05.724450 18637 tablet_copy_client.cc:783] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.724660 18637 tablet_copy_client.cc:647] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.724794 18614 ts_tablet_manager.cc:902] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:05.725271 18614 tablet_copy_client.cc:308] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:05.727146 15330 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.727787 15059 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 01:59:05.727947 15131 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6c4e85e7050b4dc8a4ce6bba818854a2. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.728183 18189 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.729243 18109 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.730659 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet 5804597cc507483dba622e6ec44e5ead from peer 3b8d615c71e44ff887e0f037c3efcff7 ({username='slave'} at 127.7.221.72:57673)
I20240502 01:59:05.730769 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet 5804597cc507483dba622e6ec44e5ead from peer 3b8d615c71e44ff887e0f037c3efcff7 at {username='slave'} at 127.7.221.72:57673: session id = 3b8d615c71e44ff887e0f037c3efcff7-5804597cc507483dba622e6ec44e5ead
I20240502 01:59:05.731318 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:05.731750 15216 tablet_copy_source_session.cc:205] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.732579 18634 raft_consensus.cc:348] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.732810 18634 raft_consensus.cc:729] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d4e5422f66454511ae5a50ec6ab134ad, State: Initialized, Role: LEARNER
I20240502 01:59:05.732975 18634 consensus_queue.cc:260] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.733376 18634 ts_tablet_manager.cc:1398] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:05.734319 15488 tablet_copy_service.cc:342] P 7b66a8940fe54e21bec1b2db25b2fed5: Request end of tablet copy session d4e5422f66454511ae5a50ec6ab134ad-fc07e430e3e74e3885f263881bd04976 received from {username='slave'} at 127.7.221.70:44607
I20240502 01:59:05.734397 15488 tablet_copy_service.cc:433] P 7b66a8940fe54e21bec1b2db25b2fed5: ending tablet copy session d4e5422f66454511ae5a50ec6ab134ad-fc07e430e3e74e3885f263881bd04976 on tablet fc07e430e3e74e3885f263881bd04976 with peer d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:05.734524 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:05.737506 18634 ts_tablet_manager.cc:902] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:05.737941 18634 tablet_copy_client.cc:308] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:05.737979 18654 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 856f623374824d94913db0cb9faa25d1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.738415 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet 81c8fe21ec7545e79450532b43a61ee7 from peer d4e5422f66454511ae5a50ec6ab134ad ({username='slave'} at 127.7.221.70:59193)
I20240502 01:59:05.738510 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet 81c8fe21ec7545e79450532b43a61ee7 from peer d4e5422f66454511ae5a50ec6ab134ad at {username='slave'} at 127.7.221.70:59193: session id = d4e5422f66454511ae5a50ec6ab134ad-81c8fe21ec7545e79450532b43a61ee7
I20240502 01:59:05.739337 15216 tablet_copy_source_session.cc:205] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.739976 18557 tablet_bootstrap.cc:492] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:05.740379 18634 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81c8fe21ec7545e79450532b43a61ee7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.740923 18654 tablet_copy_client.cc:783] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.741211 18654 tablet_copy_client.cc:647] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.742621 18634 tablet_copy_client.cc:783] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.742848 18634 tablet_copy_client.cc:647] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.743162 18557 tablet_bootstrap.cc:492] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:05.743378 18557 ts_tablet_manager.cc:1367] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.067s user 0.028s sys 0.000s
I20240502 01:59:05.743861 18456 tablet_bootstrap.cc:492] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:05.743961 18557 raft_consensus.cc:348] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:05.744134 18557 raft_consensus.cc:729] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:05.744161 18456 tablet_bootstrap.cc:492] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap complete.
I20240502 01:59:05.744324 18456 ts_tablet_manager.cc:1367] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad: Time spent bootstrapping tablet: real 0.188s user 0.025s sys 0.000s
I20240502 01:59:05.744288 18557 consensus_queue.cc:260] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:05.744716 18557 ts_tablet_manager.cc:1398] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:05.745656 15081 tablet_copy_service.cc:342] P ddae2a636423489693c12b7d7d166c09: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-8ddc239a297a46e48e069b985cb7b350 received from {username='slave'} at 127.7.221.69:51825
I20240502 01:59:05.745743 15081 tablet_copy_service.cc:433] P ddae2a636423489693c12b7d7d166c09: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-8ddc239a297a46e48e069b985cb7b350 on tablet 8ddc239a297a46e48e069b985cb7b350 with peer 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:05.746759 18614 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5804597cc507483dba622e6ec44e5ead. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.747923 18557 ts_tablet_manager.cc:902] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:05.748355 18557 tablet_copy_client.cc:308] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 01:59:05.748931 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet e2f4ca025890492880608c7975650969 from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:51343)
I20240502 01:59:05.749059 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet e2f4ca025890492880608c7975650969 from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:51343: session id = 84987ad9e4a54d96af8569e4aaf0ed64-e2f4ca025890492880608c7975650969
I20240502 01:59:05.749444 18614 tablet_copy_client.cc:783] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.749584 18614 tablet_copy_client.cc:647] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.750156 15351 tablet_copy_source_session.cc:205] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.751051 18557 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e2f4ca025890492880608c7975650969. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.753580 18557 tablet_copy_client.cc:783] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.753724 18557 tablet_copy_client.cc:647] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.754988 18557 tablet_copy_client.cc:523] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.755084 18596 tablet_bootstrap.cc:492] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
I20240502 01:59:05.755990 18557 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
I20240502 01:59:05.760478 18669 ts_tablet_manager.cc:902] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:05.762533 15465 consensus_queue.cc:237] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:05.763279 18637 tablet_copy_client.cc:523] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.763583 18614 tablet_copy_client.cc:523] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.764737 18637 tablet_bootstrap.cc:492] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
I20240502 01:59:05.765492 18614 tablet_bootstrap.cc:492] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap starting.
I20240502 01:59:05.766000 17108 raft_consensus.cc:1261] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.766588 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:05.766942 15329 raft_consensus.cc:1261] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 01:59:05.767369 15399 consensus_peers.cc:485] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 986e7c97e0734e958f364dc5d1167846. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.767385 18296 consensus_queue.cc:1035] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.768070 18296 consensus_queue.cc:1035] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.769697 15195 raft_consensus.cc:1261] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.770347 18669 tablet_copy_client.cc:308] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:05.770395 18279 consensus_queue.cc:1035] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.770824 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet 856f623374824d94913db0cb9faa25d1 from peer 3b8d615c71e44ff887e0f037c3efcff7 ({username='slave'} at 127.7.221.72:57673)
I20240502 01:59:05.770692 18279 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.770925 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet 856f623374824d94913db0cb9faa25d1 from peer 3b8d615c71e44ff887e0f037c3efcff7 at {username='slave'} at 127.7.221.72:57673: session id = 3b8d615c71e44ff887e0f037c3efcff7-856f623374824d94913db0cb9faa25d1
I20240502 01:59:05.771843 15216 tablet_copy_source_session.cc:205] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.772958 15329 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.773536 15195 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.774811 14908 catalog_manager.cc:5484] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.782023 15061 consensus_queue.cc:237] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:05.782974 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
W20240502 01:59:05.784188 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 77ae087802144c45924c1b79f561afe8. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.785988 15329 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.786551 18624 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.790251 18669 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 856f623374824d94913db0cb9faa25d1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.791296 15195 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.792407 18319 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.793006 18624 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.794291 15310 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:05.794214 15329 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.795409 14909 catalog_manager.cc:5484] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.799345 18646 tablet_copy_client.cc:523] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.799619 15195 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.801462 18646 tablet_bootstrap.cc:492] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap starting.
I20240502 01:59:05.810365 18712 ts_tablet_manager.cc:902] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:05.810946 18712 tablet_copy_client.cc:308] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 01:59:05.814004 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:05.815359 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet e2f4ca025890492880608c7975650969 from peer 3b8d615c71e44ff887e0f037c3efcff7 ({username='slave'} at 127.7.221.72:47237)
I20240502 01:59:05.815450 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet e2f4ca025890492880608c7975650969 from peer 3b8d615c71e44ff887e0f037c3efcff7 at {username='slave'} at 127.7.221.72:47237: session id = 3b8d615c71e44ff887e0f037c3efcff7-e2f4ca025890492880608c7975650969
I20240502 01:59:05.816308 15351 tablet_copy_source_session.cc:205] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.818306 18712 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e2f4ca025890492880608c7975650969. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.819859 15488 tablet_copy_service.cc:140] P 7b66a8940fe54e21bec1b2db25b2fed5: Received BeginTabletCopySession request for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:53377)
I20240502 01:59:05.819945 15488 tablet_copy_service.cc:161] P 7b66a8940fe54e21bec1b2db25b2fed5: Beginning new tablet copy session on tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:53377: session id = 84987ad9e4a54d96af8569e4aaf0ed64-fcfc8dc4b90a4a6aa7e4035bbfdb4532
I20240502 01:59:05.820716 15488 tablet_copy_source_session.cc:205] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.821328 18189 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.821758 15330 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.823136 18517 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.824748 14909 catalog_manager.cc:5484] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.825279 18517 tablet_copy_client.cc:783] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.825438 18517 tablet_copy_client.cc:647] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.827073 15059 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:05.831410 18745 ts_tablet_manager.cc:902] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8: Initiating tablet copy from peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:05.831640 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:05.831910 18745 tablet_copy_client.cc:308] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:46123
I20240502 01:59:05.833389 18654 tablet_copy_client.cc:523] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.833781 18296 raft_consensus.cc:1050] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 to VOTER
I20240502 01:59:05.834199 18296 consensus_queue.cc:237] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:05.834794 18654 tablet_bootstrap.cc:492] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap starting.
I20240502 01:59:05.835557 15330 raft_consensus.cc:1261] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
W20240502 01:59:05.837028 15399 consensus_peers.cc:485] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 986e7c97e0734e958f364dc5d1167846. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.837266 15193 raft_consensus.cc:1261] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:05.837654 18279 consensus_queue.cc:1035] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:05.747715 18634 tablet_copy_client.cc:523] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.837968 17108 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.839298 18110 consensus_queue.cc:1035] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:05.839541 18634 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap starting.
I20240502 01:59:05.839718 17106 raft_consensus.cc:1261] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:05.841598 18279 consensus_queue.cc:1035] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:05.756331 18676 tablet_copy_client.cc:523] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.843961 18279 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.844970 18676 tablet_bootstrap.cc:492] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap starting.
I20240502 01:59:05.785638 18610 tablet_copy_client.cc:523] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.846735 18610 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap starting.
I20240502 01:59:05.744737 18456 raft_consensus.cc:348] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.849110 18456 raft_consensus.cc:729] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d4e5422f66454511ae5a50ec6ab134ad, State: Initialized, Role: LEARNER
I20240502 01:59:05.849211 18456 consensus_queue.cc:260] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.849459 18456 ts_tablet_manager.cc:1398] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad: Time spent starting tablet: real 0.105s user 0.001s sys 0.000s
I20240502 01:59:05.849936 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session d4e5422f66454511ae5a50ec6ab134ad-a901eb3063ea4911a822552db01d365d received from {username='slave'} at 127.7.221.70:59193
I20240502 01:59:05.850020 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session d4e5422f66454511ae5a50ec6ab134ad-a901eb3063ea4911a822552db01d365d on tablet a901eb3063ea4911a822552db01d365d with peer d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:05.851560 15197 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.852049 18737 ts_tablet_manager.cc:902] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8: Initiating tablet copy from peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:05.853417 18737 tablet_copy_client.cc:308] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:46123
I20240502 01:59:05.854540 14915 catalog_manager.cc:5484] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.855865 15080 tablet_copy_service.cc:140] P ddae2a636423489693c12b7d7d166c09: Received BeginTabletCopySession request for tablet 1eb15147ff6c4d0b83b12dec736fc576 from peer fc067ee690bb457e8109ad8e4eb43ab8 ({username='slave'} at 127.7.221.71:53189)
I20240502 01:59:05.855911 15081 tablet_copy_service.cc:140] P ddae2a636423489693c12b7d7d166c09: Received BeginTabletCopySession request for tablet e00d1ed5cde044cb9bdebf787590261a from peer fc067ee690bb457e8109ad8e4eb43ab8 ({username='slave'} at 127.7.221.71:53189)
I20240502 01:59:05.855974 15080 tablet_copy_service.cc:161] P ddae2a636423489693c12b7d7d166c09: Beginning new tablet copy session on tablet 1eb15147ff6c4d0b83b12dec736fc576 from peer fc067ee690bb457e8109ad8e4eb43ab8 at {username='slave'} at 127.7.221.71:53189: session id = fc067ee690bb457e8109ad8e4eb43ab8-1eb15147ff6c4d0b83b12dec736fc576
I20240502 01:59:05.855983 15330 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.857002 15080 tablet_copy_source_session.cc:205] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.857301 15081 tablet_copy_service.cc:161] P ddae2a636423489693c12b7d7d166c09: Beginning new tablet copy session on tablet e00d1ed5cde044cb9bdebf787590261a from peer fc067ee690bb457e8109ad8e4eb43ab8 at {username='slave'} at 127.7.221.71:53189: session id = fc067ee690bb457e8109ad8e4eb43ab8-e00d1ed5cde044cb9bdebf787590261a
I20240502 01:59:05.858039 15081 tablet_copy_source_session.cc:205] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.858286 18669 tablet_copy_client.cc:783] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.858438 18669 tablet_copy_client.cc:647] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.859376 18745 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e00d1ed5cde044cb9bdebf787590261a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.861549 18745 tablet_copy_client.cc:783] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.861749 18745 tablet_copy_client.cc:647] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.862087 17106 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.864509 18745 tablet_copy_client.cc:523] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.865764 18745 tablet_bootstrap.cc:492] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap starting.
I20240502 01:59:05.866873 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:05.867831 16974 raft_consensus.cc:1203] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 01:59:05.869912 15465 consensus_queue.cc:237] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:05.870648 18669 tablet_copy_client.cc:523] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
W20240502 01:59:05.871723 15399 consensus_peers.cc:485] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 986e7c97e0734e958f364dc5d1167846. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.871922 18669 tablet_bootstrap.cc:492] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap starting.
I20240502 01:59:05.872452 17106 raft_consensus.cc:1261] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:05.872809 18296 consensus_queue.cc:1035] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:05.874143 15330 raft_consensus.cc:1261] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:05.874317 18296 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.874487 18279 consensus_queue.cc:1035] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:05.874925 18779 tablet_replica.cc:330] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: stopping tablet replica
I20240502 01:59:05.875293 18779 raft_consensus.cc:2229] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Raft consensus shutting down.
I20240502 01:59:05.875380 15331 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.875583 18779 raft_consensus.cc:2258] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Raft consensus is shut down!
I20240502 01:59:05.876255 17106 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.877080 18779 ts_tablet_manager.cc:1875] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:05.878196 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 986e7c97e0734e958f364dc5d1167846 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:05.878895 14908 catalog_manager.cc:5484] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 6 to 7, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:05.881539 15174 tablet_service.cc:1514] Processing DeleteTablet for tablet 986e7c97e0734e958f364dc5d1167846 with delete_type TABLET_DATA_TOMBSTONED (TS 26a3d5ced97d4dcc919b80fde53f6a5b not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:37206
I20240502 01:59:05.890838 18782 ts_tablet_manager.cc:902] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:05.891114 18783 tablet_replica.cc:330] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b: stopping tablet replica
I20240502 01:59:05.891420 18783 raft_consensus.cc:2229] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:05.891537 18782 tablet_copy_client.cc:308] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:46123
I20240502 01:59:05.891666 18783 raft_consensus.cc:2258] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:05.893101 18783 ts_tablet_manager.cc:1875] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:05.893229 18779 ts_tablet_manager.cc:1888] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.4
I20240502 01:59:05.893347 18779 log.cc:1196] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal/wals/81c8fe21ec7545e79450532b43a61ee7
I20240502 01:59:05.893723 18779 ts_tablet_manager.cc:902] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:05.894276 18779 tablet_copy_client.cc:308] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:05.894779 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet 81c8fe21ec7545e79450532b43a61ee7 from peer 3b8d615c71e44ff887e0f037c3efcff7 ({username='slave'} at 127.7.221.72:57673)
I20240502 01:59:05.894881 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet 81c8fe21ec7545e79450532b43a61ee7 from peer 3b8d615c71e44ff887e0f037c3efcff7 at {username='slave'} at 127.7.221.72:57673: session id = 3b8d615c71e44ff887e0f037c3efcff7-81c8fe21ec7545e79450532b43a61ee7
I20240502 01:59:05.895677 15216 tablet_copy_source_session.cc:205] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.897903 16838 raft_consensus.cc:1203] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 01:59:05.898671 18779 ts_tablet_manager.cc:1875] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 01:59:05.898855 18783 ts_tablet_manager.cc:1888] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 01:59:05.899034 18783 log.cc:1196] T 986e7c97e0734e958f364dc5d1167846 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/986e7c97e0734e958f364dc5d1167846
I20240502 01:59:05.900370 15081 tablet_copy_service.cc:140] P ddae2a636423489693c12b7d7d166c09: Received BeginTabletCopySession request for tablet 1eb15147ff6c4d0b83b12dec736fc576 from peer 3b8d615c71e44ff887e0f037c3efcff7 ({username='slave'} at 127.7.221.72:45525)
I20240502 01:59:05.900477 15081 tablet_copy_service.cc:161] P ddae2a636423489693c12b7d7d166c09: Beginning new tablet copy session on tablet 1eb15147ff6c4d0b83b12dec736fc576 from peer 3b8d615c71e44ff887e0f037c3efcff7 at {username='slave'} at 127.7.221.72:45525: session id = 3b8d615c71e44ff887e0f037c3efcff7-1eb15147ff6c4d0b83b12dec736fc576
I20240502 01:59:05.900799 18779 ts_tablet_manager.cc:1888] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.4
I20240502 01:59:05.900996 18779 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81c8fe21ec7545e79450532b43a61ee7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.901293 15081 tablet_copy_source_session.cc:205] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.902083 18779 tablet_copy_client.cc:783] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.902258 14900 catalog_manager.cc:4830] TS 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905): tablet 986e7c97e0734e958f364dc5d1167846 (table rebalance_test_table_2 [id=73bf3d4779b34c7290b42f7f561e8e11]) successfully deleted
I20240502 01:59:05.902530 18779 tablet_copy_client.cc:647] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.906169 15330 consensus_queue.cc:237] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:05.907796 18712 tablet_copy_client.cc:783] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.908022 18712 tablet_copy_client.cc:647] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 1 WAL segments...
W20240502 01:59:05.908152 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ce85785e670a4264b44797bb27cb4ecb. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.908764 15465 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.909289 18417 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.909463 18614 tablet_bootstrap.cc:492] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:05.909689 18614 tablet_bootstrap.cc:492] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap complete.
I20240502 01:59:05.909840 18614 ts_tablet_manager.cc:1367] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7: Time spent bootstrapping tablet: real 0.144s user 0.025s sys 0.000s
I20240502 01:59:05.910488 18614 raft_consensus.cc:348] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:05.910629 18614 raft_consensus.cc:729] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b8d615c71e44ff887e0f037c3efcff7, State: Initialized, Role: LEARNER
I20240502 01:59:05.910727 18614 consensus_queue.cc:260] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:05.913097 18614 ts_tablet_manager.cc:1398] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 01:59:05.913219 18779 tablet_copy_client.cc:523] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.913764 18712 tablet_copy_client.cc:523] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.915130 18712 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap starting.
I20240502 01:59:05.915416 18711 ts_tablet_manager.cc:902] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:05.915953 18711 tablet_copy_client.cc:308] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:34323
I20240502 01:59:05.915961 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-5804597cc507483dba622e6ec44e5ead received from {username='slave'} at 127.7.221.72:57673
I20240502 01:59:05.916052 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-5804597cc507483dba622e6ec44e5ead on tablet 5804597cc507483dba622e6ec44e5ead with peer 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:05.916718 15488 tablet_copy_service.cc:140] P 7b66a8940fe54e21bec1b2db25b2fed5: Received BeginTabletCopySession request for tablet 6d5cffb7fcc54118861463e3b1fa4a8d from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:53377)
I20240502 01:59:05.916842 15488 tablet_copy_service.cc:161] P 7b66a8940fe54e21bec1b2db25b2fed5: Beginning new tablet copy session on tablet 6d5cffb7fcc54118861463e3b1fa4a8d from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:53377: session id = 84987ad9e4a54d96af8569e4aaf0ed64-6d5cffb7fcc54118861463e3b1fa4a8d
I20240502 01:59:05.917002 18779 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap starting.
I20240502 01:59:05.917850 15488 tablet_copy_source_session.cc:205] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:05.918412 15059 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.918957 18417 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.920555 18782 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1eb15147ff6c4d0b83b12dec736fc576. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.920576 18711 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6d5cffb7fcc54118861463e3b1fa4a8d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.922683 18782 tablet_copy_client.cc:783] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.922904 18782 tablet_copy_client.cc:647] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.922581 18711 tablet_copy_client.cc:783] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.923353 18711 tablet_copy_client.cc:647] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.930367 18782 tablet_copy_client.cc:523] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.931777 18782 tablet_bootstrap.cc:492] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap starting.
I20240502 01:59:05.932906 18711 tablet_copy_client.cc:523] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.934171 18711 tablet_bootstrap.cc:492] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
I20240502 01:59:05.934330 15195 consensus_queue.cc:237] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.936136 17244 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.937361 15465 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.937724 18109 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 01:59:05.938302 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 5804597cc507483dba622e6ec44e5ead. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.938449 18109 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.942188 15330 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.942613 18109 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.944137 16838 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.944636 18109 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.946159 18596 tablet_bootstrap.cc:492] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:05.947492 15331 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.948278 14909 catalog_manager.cc:5484] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.948539 15465 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.949942 18517 tablet_copy_client.cc:523] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:05.951440 18517 tablet_bootstrap.cc:492] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
I20240502 01:59:05.956734 15464 consensus_queue.cc:237] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.956750 18737 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1eb15147ff6c4d0b83b12dec736fc576. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:05.957995 15061 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.958329 18596 tablet_bootstrap.cc:492] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:05.958492 18279 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.958549 18596 ts_tablet_manager.cc:1367] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.204s user 0.023s sys 0.006s
W20240502 01:59:05.958950 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3f938c5af51a43c9a23b753c4991e41d. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.959189 18596 raft_consensus.cc:348] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:05.959384 18596 raft_consensus.cc:729] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:05.959533 18596 consensus_queue.cc:260] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 1.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:05.959952 18596 ts_tablet_manager.cc:1398] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:05.960559 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-856f623374824d94913db0cb9faa25d1 received from {username='slave'} at 127.7.221.69:43177
I20240502 01:59:05.960628 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-856f623374824d94913db0cb9faa25d1 on tablet 856f623374824d94913db0cb9faa25d1 with peer 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:05.961151 15195 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:05.962198 18766 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:05.968848 15197 consensus_queue.cc:237] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:05.971144 15060 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 01:59:05.971879 15127 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6c4e85e7050b4dc8a4ce6bba818854a2. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.972177 18189 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 01:59:05.974762 15131 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6c4e85e7050b4dc8a4ce6bba818854a2. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:05.976689 18737 tablet_copy_client.cc:783] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:05.976917 18737 tablet_copy_client.cc:647] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:05.976802 18189 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.979075 15060 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.979311 18557 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:05.979554 18557 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:05.979701 18557 ts_tablet_manager.cc:1367] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.224s user 0.023s sys 0.003s
I20240502 01:59:05.980139 18557 raft_consensus.cc:348] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:05.980264 18557 raft_consensus.cc:729] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:05.980346 18557 consensus_queue.cc:260] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 1.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:05.981005 18557 ts_tablet_manager.cc:1398] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 01:59:05.981839 14915 catalog_manager.cc:5484] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.982280 18637 tablet_bootstrap.cc:492] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:05.982561 18637 tablet_bootstrap.cc:492] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:05.982581 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-e2f4ca025890492880608c7975650969 received from {username='slave'} at 127.7.221.69:51343
I20240502 01:59:05.982693 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-e2f4ca025890492880608c7975650969 on tablet e2f4ca025890492880608c7975650969 with peer 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:05.982818 18637 ts_tablet_manager.cc:1367] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.218s user 0.026s sys 0.002s
I20240502 01:59:05.983237 15330 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:05.983204 18637 raft_consensus.cc:348] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:05.983350 18637 raft_consensus.cc:729] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:05.983500 18637 consensus_queue.cc:260] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:05.983717 18109 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:05.983793 18637 ts_tablet_manager.cc:1398] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:59:05.984354 15081 tablet_copy_service.cc:342] P ddae2a636423489693c12b7d7d166c09: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-e00d1ed5cde044cb9bdebf787590261a received from {username='slave'} at 127.7.221.69:51825
I20240502 01:59:05.984449 15081 tablet_copy_service.cc:433] P ddae2a636423489693c12b7d7d166c09: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-e00d1ed5cde044cb9bdebf787590261a on tablet e00d1ed5cde044cb9bdebf787590261a with peer 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:05.985249 15330 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:05.985792 18767 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.989377 14908 catalog_manager.cc:5484] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:05.991087 15061 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:05.992858 16974 raft_consensus.cc:1203] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:05.993760 16974 raft_consensus.cc:1203] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:05.996474 18676 tablet_bootstrap.cc:492] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:05.996958 18676 tablet_bootstrap.cc:492] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap complete.
I20240502 01:59:05.997174 18676 ts_tablet_manager.cc:1367] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad: Time spent bootstrapping tablet: real 0.153s user 0.025s sys 0.000s
I20240502 01:59:05.997624 18676 raft_consensus.cc:348] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.997794 18676 raft_consensus.cc:729] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d4e5422f66454511ae5a50ec6ab134ad, State: Initialized, Role: LEARNER
I20240502 01:59:05.997931 18676 consensus_queue.cc:260] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.998315 18676 ts_tablet_manager.cc:1398] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:05.998981 18417 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.001819 14908 catalog_manager.cc:5484] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:06.003645 15464 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.006230 18818 ts_tablet_manager.cc:902] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:06.006940 18818 tablet_copy_client.cc:308] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:34323
I20240502 01:59:06.007495 15488 tablet_copy_service.cc:140] P 7b66a8940fe54e21bec1b2db25b2fed5: Received BeginTabletCopySession request for tablet 6d5cffb7fcc54118861463e3b1fa4a8d from peer fc067ee690bb457e8109ad8e4eb43ab8 ({username='slave'} at 127.7.221.71:50737)
I20240502 01:59:06.007603 15488 tablet_copy_service.cc:161] P 7b66a8940fe54e21bec1b2db25b2fed5: Beginning new tablet copy session on tablet 6d5cffb7fcc54118861463e3b1fa4a8d from peer fc067ee690bb457e8109ad8e4eb43ab8 at {username='slave'} at 127.7.221.71:50737: session id = fc067ee690bb457e8109ad8e4eb43ab8-6d5cffb7fcc54118861463e3b1fa4a8d
I20240502 01:59:06.008520 15488 tablet_copy_source_session.cc:205] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.008970 18446 tablet_bootstrap.cc:492] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.009199 18446 tablet_bootstrap.cc:492] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap complete.
I20240502 01:59:06.009347 18446 ts_tablet_manager.cc:1367] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8: Time spent bootstrapping tablet: real 0.439s user 0.025s sys 0.000s
I20240502 01:59:06.009686 17910 raft_consensus.cc:1050] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 to VOTER
I20240502 01:59:06.009919 18446 raft_consensus.cc:348] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.010044 18446 raft_consensus.cc:729] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fc067ee690bb457e8109ad8e4eb43ab8, State: Initialized, Role: LEARNER
I20240502 01:59:06.009950 15059 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.010140 18446 consensus_queue.cc:260] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.010162 17910 consensus_queue.cc:237] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:06.012341 15331 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.012941 18189 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.013134 15464 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.013784 17242 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.014189 17244 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.017242 18189 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.019125 18367 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.019294 18460 tablet_bootstrap.cc:492] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.019543 18460 tablet_bootstrap.cc:492] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap complete.
I20240502 01:59:06.019707 18460 ts_tablet_manager.cc:1367] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7: Time spent bootstrapping tablet: real 0.510s user 0.027s sys 0.000s
I20240502 01:59:06.020136 18460 raft_consensus.cc:348] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:06.020246 18460 raft_consensus.cc:729] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b8d615c71e44ff887e0f037c3efcff7, State: Initialized, Role: LEARNER
I20240502 01:59:06.020159 18367 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.020334 18460 consensus_queue.cc:260] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:06.020584 18460 ts_tablet_manager.cc:1398] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:06.021059 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-a901eb3063ea4911a822552db01d365d received from {username='slave'} at 127.7.221.72:57673
I20240502 01:59:06.021122 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-a901eb3063ea4911a822552db01d365d on tablet a901eb3063ea4911a822552db01d365d with peer 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:06.022655 15331 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.022653 15464 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.025277 14908 catalog_manager.cc:5484] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.025616 18637 ts_tablet_manager.cc:902] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:06.026078 18637 tablet_copy_client.cc:308] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:46123
I20240502 01:59:06.026708 15081 tablet_copy_service.cc:140] P ddae2a636423489693c12b7d7d166c09: Received BeginTabletCopySession request for tablet d1c832c721fb4b15bc6d266a1c295401 from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:51825)
I20240502 01:59:06.026814 15081 tablet_copy_service.cc:161] P ddae2a636423489693c12b7d7d166c09: Beginning new tablet copy session on tablet d1c832c721fb4b15bc6d266a1c295401 from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:51825: session id = 84987ad9e4a54d96af8569e4aaf0ed64-d1c832c721fb4b15bc6d266a1c295401
I20240502 01:59:06.027329 18818 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6d5cffb7fcc54118861463e3b1fa4a8d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.027695 15081 tablet_copy_source_session.cc:205] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.028515 18637 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d1c832c721fb4b15bc6d266a1c295401. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.031488 18637 tablet_copy_client.cc:783] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.031873 18637 tablet_copy_client.cc:647] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.035315 15197 consensus_queue.cc:237] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:06.035138 17244 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.036772 18711 tablet_bootstrap.cc:492] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.037124 18711 tablet_bootstrap.cc:492] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:06.037163 15331 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.037415 18711 ts_tablet_manager.cc:1367] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.103s user 0.022s sys 0.004s
I20240502 01:59:06.037865 18711 raft_consensus.cc:348] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.038380 18711 raft_consensus.cc:729] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:06.038551 18711 consensus_queue.cc:260] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.038630 17242 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.039211 18189 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.039487 18711 ts_tablet_manager.cc:1398] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:06.039496 18189 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.039999 15488 tablet_copy_service.cc:342] P 7b66a8940fe54e21bec1b2db25b2fed5: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-6d5cffb7fcc54118861463e3b1fa4a8d received from {username='slave'} at 127.7.221.69:53377
I20240502 01:59:06.040081 15488 tablet_copy_service.cc:433] P 7b66a8940fe54e21bec1b2db25b2fed5: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-6d5cffb7fcc54118861463e3b1fa4a8d on tablet 6d5cffb7fcc54118861463e3b1fa4a8d with peer 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:06.040993 15464 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.042541 15195 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.042894 17471 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.045847 18189 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:06.047369 18584 tablet_bootstrap.cc:492] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.047655 18584 tablet_bootstrap.cc:492] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap complete.
I20240502 01:59:06.047746 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 5804597cc507483dba622e6ec44e5ead with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.047832 18584 ts_tablet_manager.cc:1367] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad: Time spent bootstrapping tablet: real 0.445s user 0.024s sys 0.005s
I20240502 01:59:06.047895 17242 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:06.048409 15331 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:06.048516 15465 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:06.051492 14913 catalog_manager.cc:5484] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 6 to 7, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:06.054198 18737 tablet_copy_client.cc:523] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.055752 15195 consensus_queue.cc:237] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:06.056124 18737 tablet_bootstrap.cc:492] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap starting.
I20240502 01:59:06.059069 15465 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.059459 17910 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.060056 17244 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.060410 17910 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.061087 18109 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:06.061460 15465 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:06.062629 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 5804597cc507483dba622e6ec44e5ead with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.063907 14909 catalog_manager.cc:5484] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:06.065284 16954 tablet_service.cc:1514] Processing DeleteTablet for tablet 5804597cc507483dba622e6ec44e5ead with delete_type TABLET_DATA_TOMBSTONED (TS d4e5422f66454511ae5a50ec6ab134ad not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:52144
I20240502 01:59:06.066207 17244 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:06.067552 18446 ts_tablet_manager.cc:1398] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8: Time spent starting tablet: real 0.058s user 0.002s sys 0.000s
I20240502 01:59:06.068733 15059 consensus_queue.cc:237] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:06.068188 18818 tablet_copy_client.cc:783] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.070502 15195 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.070703 18818 tablet_copy_client.cc:647] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.071274 15330 consensus_queue.cc:237] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.072094 15488 tablet_copy_service.cc:342] P 7b66a8940fe54e21bec1b2db25b2fed5: Request end of tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-ac5ec2c673cb4b988d1099e0442ec63f received from {username='slave'} at 127.7.221.71:50737
I20240502 01:59:06.072188 15488 tablet_copy_service.cc:433] P 7b66a8940fe54e21bec1b2db25b2fed5: ending tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-ac5ec2c673cb4b988d1099e0442ec63f on tablet ac5ec2c673cb4b988d1099e0442ec63f with peer fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:06.073685 15465 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.074036 18584 raft_consensus.cc:348] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
W20240502 01:59:06.074373 15266 consensus_peers.cc:485] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 091d27d51acc4db0a1c7d20deeff5c10. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.074538 15059 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.074374 18071 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.074769 15331 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.075254 18386 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.075497 18386 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.076504 18584 raft_consensus.cc:729] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d4e5422f66454511ae5a50ec6ab134ad, State: Initialized, Role: LEARNER
I20240502 01:59:06.076635 18584 consensus_queue.cc:260] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:06.077059 15310 tablet_service.cc:1514] Processing DeleteTablet for tablet 5804597cc507483dba622e6ec44e5ead with delete_type TABLET_DATA_TOMBSTONED (TS f77c2a969bde45b49ac1bbbe242ff171 not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:43766
I20240502 01:59:06.079782 18818 tablet_copy_client.cc:523] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.080586 18446 ts_tablet_manager.cc:902] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:06.080977 18071 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.081173 18446 tablet_copy_client.cc:308] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:06.082408 18818 tablet_bootstrap.cc:492] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap starting.
I20240502 01:59:06.083418 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet d24986c2ebab46f086e3f1f5a53af26b from peer fc067ee690bb457e8109ad8e4eb43ab8 ({username='slave'} at 127.7.221.71:39719)
I20240502 01:59:06.083515 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet d24986c2ebab46f086e3f1f5a53af26b from peer fc067ee690bb457e8109ad8e4eb43ab8 at {username='slave'} at 127.7.221.71:39719: session id = fc067ee690bb457e8109ad8e4eb43ab8-d24986c2ebab46f086e3f1f5a53af26b
I20240502 01:59:06.084411 15216 tablet_copy_source_session.cc:205] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.085178 18446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d24986c2ebab46f086e3f1f5a53af26b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.085232 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session d4e5422f66454511ae5a50ec6ab134ad-30620f4774b8414eb4e27bd5ecb6ecee received from {username='slave'} at 127.7.221.70:33955
I20240502 01:59:06.085311 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session d4e5422f66454511ae5a50ec6ab134ad-30620f4774b8414eb4e27bd5ecb6ecee on tablet 30620f4774b8414eb4e27bd5ecb6ecee with peer d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:06.087042 18071 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.087663 15331 consensus_queue.cc:237] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.089900 15061 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.089999 18624 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.090234 15465 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.090904 15464 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 01:59:06.091446 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ce85785e670a4264b44797bb27cb4ecb. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.091581 16972 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.092700 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d1c832c721fb4b15bc6d266a1c295401 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.092867 18319 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.093050 15059 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.093829 16972 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.094168 14909 catalog_manager.cc:5484] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 8 to 9, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:06.094803 18584 ts_tablet_manager.cc:1398] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad: Time spent starting tablet: real 0.047s user 0.002s sys 0.000s
I20240502 01:59:06.095274 15488 tablet_copy_service.cc:342] P 7b66a8940fe54e21bec1b2db25b2fed5: Request end of tablet copy session d4e5422f66454511ae5a50ec6ab134ad-bf73e677bcd049869cadf9f8c9cd3434 received from {username='slave'} at 127.7.221.70:44607
I20240502 01:59:06.095345 15488 tablet_copy_service.cc:433] P 7b66a8940fe54e21bec1b2db25b2fed5: ending tablet copy session d4e5422f66454511ae5a50ec6ab134ad-bf73e677bcd049869cadf9f8c9cd3434 on tablet bf73e677bcd049869cadf9f8c9cd3434 with peer d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:06.096160 15195 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.098309 16838 raft_consensus.cc:1203] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.5->[1.6-1.6] Dedup: 1.6->[]
I20240502 01:59:06.100035 18851 tablet_replica.cc:330] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: stopping tablet replica
I20240502 01:59:06.100973 14915 catalog_manager.cc:5484] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:06.101878 18446 tablet_copy_client.cc:783] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.102082 18446 tablet_copy_client.cc:647] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 1 WAL segments...
W20240502 01:59:06.102350 15265 consensus_peers.cc:485] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 091d27d51acc4db0a1c7d20deeff5c10. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.102839 18417 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.103176 18417 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.104188 18851 raft_consensus.cc:2229] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:06.105952 16838 raft_consensus.cc:1203] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.5->[1.6-1.6] Dedup: 1.6->[]
I20240502 01:59:06.106534 16820 tablet_service.cc:1514] Processing DeleteTablet for tablet d1c832c721fb4b15bc6d266a1c295401 with delete_type TABLET_DATA_TOMBSTONED (TS 84987ad9e4a54d96af8569e4aaf0ed64 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:55224
I20240502 01:59:06.109354 15465 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.109664 15059 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.112157 14909 catalog_manager.cc:5484] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.107084 15331 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.112910 18851 raft_consensus.cc:2258] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:06.108981 18417 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.120867 18851 ts_tablet_manager.cc:1875] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.121803 15331 consensus_queue.cc:237] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.123376 18851 ts_tablet_manager.cc:1888] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:06.123445 15465 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.123735 15195 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.124454 16972 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.125389 18446 tablet_copy_client.cc:523] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.126569 18845 ts_tablet_manager.cc:902] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8: Initiating tablet copy from peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:06.126816 18446 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap starting.
I20240502 01:59:06.130517 18779 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.130920 18779 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap complete.
I20240502 01:59:06.131112 18779 ts_tablet_manager.cc:1367] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Time spent bootstrapping tablet: real 0.214s user 0.023s sys 0.005s
I20240502 01:59:06.131683 18779 raft_consensus.cc:348] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:06.131835 18779 raft_consensus.cc:729] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b8d615c71e44ff887e0f037c3efcff7, State: Initialized, Role: LEARNER
I20240502 01:59:06.131978 18779 consensus_queue.cc:260] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:06.132815 18779 ts_tablet_manager.cc:1398] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:06.133349 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-81c8fe21ec7545e79450532b43a61ee7 received from {username='slave'} at 127.7.221.72:57673
I20240502 01:59:06.133428 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-81c8fe21ec7545e79450532b43a61ee7 on tablet 81c8fe21ec7545e79450532b43a61ee7 with peer 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:06.134833 18845 tablet_copy_client.cc:308] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:46123
I20240502 01:59:06.135349 15081 tablet_copy_service.cc:140] P ddae2a636423489693c12b7d7d166c09: Received BeginTabletCopySession request for tablet 53edbd87d0d5471ca7626ca2071f75eb from peer fc067ee690bb457e8109ad8e4eb43ab8 ({username='slave'} at 127.7.221.71:53189)
I20240502 01:59:06.135468 15081 tablet_copy_service.cc:161] P ddae2a636423489693c12b7d7d166c09: Beginning new tablet copy session on tablet 53edbd87d0d5471ca7626ca2071f75eb from peer fc067ee690bb457e8109ad8e4eb43ab8 at {username='slave'} at 127.7.221.71:53189: session id = fc067ee690bb457e8109ad8e4eb43ab8-53edbd87d0d5471ca7626ca2071f75eb
I20240502 01:59:06.136516 15081 tablet_copy_source_session.cc:205] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.137396 18845 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 53edbd87d0d5471ca7626ca2071f75eb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.139811 18845 tablet_copy_client.cc:783] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.140019 18845 tablet_copy_client.cc:647] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.141059 18845 tablet_copy_client.cc:523] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.142273 18845 tablet_bootstrap.cc:492] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap starting.
W20240502 01:59:06.143180 14901 catalog_manager.cc:4797] TS d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): delete failed for tablet 5804597cc507483dba622e6ec44e5ead because the tablet was not found. No further retry: Not found: Tablet not found: 5804597cc507483dba622e6ec44e5ead
I20240502 01:59:06.150817 18637 tablet_copy_client.cc:523] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.152175 18637 tablet_bootstrap.cc:492] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
W20240502 01:59:06.156239 14902 catalog_manager.cc:4810] TS 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): delete failed for tablet d1c832c721fb4b15bc6d266a1c295401 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet d1c832c721fb4b15bc6d266a1c295401 already in progress: copying tablet
I20240502 01:59:06.158353 18446 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.161556 18646 tablet_bootstrap.cc:492] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.168351 18669 tablet_bootstrap.cc:492] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.168911 18646 tablet_bootstrap.cc:492] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap complete.
I20240502 01:59:06.124212 18417 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 01:59:06.148447 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ce85785e670a4264b44797bb27cb4ecb. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.169145 18416 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.169433 18851 log.cc:1196] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/5804597cc507483dba622e6ec44e5ead
I20240502 01:59:06.169687 18646 ts_tablet_manager.cc:1367] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8: Time spent bootstrapping tablet: real 0.368s user 0.026s sys 0.002s
I20240502 01:59:06.170117 18071 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 01:59:06.171809 15266 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 30620f4774b8414eb4e27bd5ecb6ecee. This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:06.171885 15266 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.7.221.72:37645, user_credentials={real_user=slave}} blocked reactor thread for 46815.1us
I20240502 01:59:06.173353 18446 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap complete.
I20240502 01:59:06.175180 18446 ts_tablet_manager.cc:1367] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8: Time spent bootstrapping tablet: real 0.048s user 0.025s sys 0.000s
I20240502 01:59:06.175103 18417 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.175808 18446 raft_consensus.cc:348] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.176003 18446 raft_consensus.cc:729] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fc067ee690bb457e8109ad8e4eb43ab8, State: Initialized, Role: LEARNER
I20240502 01:59:06.176167 18446 consensus_queue.cc:260] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.177129 18446 ts_tablet_manager.cc:1398] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:06.177664 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-d24986c2ebab46f086e3f1f5a53af26b received from {username='slave'} at 127.7.221.71:39719
I20240502 01:59:06.177758 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-d24986c2ebab46f086e3f1f5a53af26b on tablet d24986c2ebab46f086e3f1f5a53af26b with peer fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:06.178846 14913 catalog_manager.cc:5484] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:06.179517 18646 raft_consensus.cc:348] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.179632 18646 raft_consensus.cc:729] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fc067ee690bb457e8109ad8e4eb43ab8, State: Initialized, Role: LEARNER
I20240502 01:59:06.179730 18646 consensus_queue.cc:260] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.179780 15465 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.168707 18669 tablet_bootstrap.cc:492] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap complete.
I20240502 01:59:06.181916 18646 ts_tablet_manager.cc:1398] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240502 01:59:06.182008 18669 ts_tablet_manager.cc:1367] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7: Time spent bootstrapping tablet: real 0.310s user 0.024s sys 0.003s
I20240502 01:59:06.182034 17244 raft_consensus.cc:1203] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:06.182344 15488 tablet_copy_service.cc:342] P 7b66a8940fe54e21bec1b2db25b2fed5: Request end of tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-fcfc8dc4b90a4a6aa7e4035bbfdb4532 received from {username='slave'} at 127.7.221.71:50737
I20240502 01:59:06.182417 15488 tablet_copy_service.cc:433] P 7b66a8940fe54e21bec1b2db25b2fed5: ending tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-fcfc8dc4b90a4a6aa7e4035bbfdb4532 on tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 with peer fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:06.182449 18669 raft_consensus.cc:348] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.182572 18669 raft_consensus.cc:729] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b8d615c71e44ff887e0f037c3efcff7, State: Initialized, Role: LEARNER
I20240502 01:59:06.174539 18590 tablet_bootstrap.cc:492] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.184250 18863 raft_consensus.cc:1050] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171: attempting to promote NON_VOTER d4e5422f66454511ae5a50ec6ab134ad to VOTER
I20240502 01:59:06.184365 14900 catalog_manager.cc:4830] TS f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): tablet 5804597cc507483dba622e6ec44e5ead (table rebalance_test_table_3 [id=0b29d58fe9c74d1cb4345bb6cfbd0535]) successfully deleted
I20240502 01:59:06.184823 18863 consensus_queue.cc:237] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
W20240502 01:59:06.187273 15266 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 30620f4774b8414eb4e27bd5ecb6ecee. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.187460 15465 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.189527 16972 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.191963 16971 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.192412 18417 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.194227 18417 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.196630 18417 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.182700 18669 consensus_queue.cc:260] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 1.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.199347 15465 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.199666 18845 tablet_bootstrap.cc:492] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.199573 14915 catalog_manager.cc:5484] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:06.200170 18845 tablet_bootstrap.cc:492] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap complete.
I20240502 01:59:06.200440 18845 ts_tablet_manager.cc:1367] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8: Time spent bootstrapping tablet: real 0.058s user 0.022s sys 0.004s
I20240502 01:59:06.201027 18845 raft_consensus.cc:348] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.201236 18845 raft_consensus.cc:729] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fc067ee690bb457e8109ad8e4eb43ab8, State: Initialized, Role: LEARNER
I20240502 01:59:06.201395 18845 consensus_queue.cc:260] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.202221 18845 ts_tablet_manager.cc:1398] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:06.202733 16971 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.203137 15081 tablet_copy_service.cc:342] P ddae2a636423489693c12b7d7d166c09: Request end of tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-53edbd87d0d5471ca7626ca2071f75eb received from {username='slave'} at 127.7.221.71:53189
I20240502 01:59:06.203227 15081 tablet_copy_service.cc:433] P ddae2a636423489693c12b7d7d166c09: ending tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-53edbd87d0d5471ca7626ca2071f75eb on tablet 53edbd87d0d5471ca7626ca2071f75eb with peer fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:06.204509 18669 ts_tablet_manager.cc:1398] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7: Time spent starting tablet: real 0.022s user 0.001s sys 0.000s
I20240502 01:59:06.205268 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-856f623374824d94913db0cb9faa25d1 received from {username='slave'} at 127.7.221.72:57673
I20240502 01:59:06.205353 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-856f623374824d94913db0cb9faa25d1 on tablet 856f623374824d94913db0cb9faa25d1 with peer 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:06.211582 18590 tablet_bootstrap.cc:492] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap complete.
I20240502 01:59:06.211819 18590 ts_tablet_manager.cc:1367] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7: Time spent bootstrapping tablet: real 0.528s user 0.023s sys 0.004s
I20240502 01:59:06.212312 18590 raft_consensus.cc:348] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.212539 18590 raft_consensus.cc:729] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b8d615c71e44ff887e0f037c3efcff7, State: Initialized, Role: LEARNER
I20240502 01:59:06.212715 18590 consensus_queue.cc:260] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.219363 18634 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.219681 18634 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap complete.
I20240502 01:59:06.219906 18634 ts_tablet_manager.cc:1367] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad: Time spent bootstrapping tablet: real 0.380s user 0.024s sys 0.003s
I20240502 01:59:06.220356 18634 raft_consensus.cc:348] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:06.221623 15193 consensus_queue.cc:237] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.221858 18634 raft_consensus.cc:729] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d4e5422f66454511ae5a50ec6ab134ad, State: Initialized, Role: LEARNER
I20240502 01:59:06.222014 18634 consensus_queue.cc:260] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:06.223018 18634 ts_tablet_manager.cc:1398] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
W20240502 01:59:06.223102 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cd00409d47954ee2b7555e7ff09a6e28. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.223798 15331 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:06.224160 18367 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:06.224242 15059 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:06.224761 17910 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:06.225308 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session d4e5422f66454511ae5a50ec6ab134ad-81c8fe21ec7545e79450532b43a61ee7 received from {username='slave'} at 127.7.221.70:59193
I20240502 01:59:06.225399 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session d4e5422f66454511ae5a50ec6ab134ad-81c8fe21ec7545e79450532b43a61ee7 on tablet 81c8fe21ec7545e79450532b43a61ee7 with peer d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:06.228904 18610 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.229300 18517 tablet_bootstrap.cc:492] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.229504 18610 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap complete.
I20240502 01:59:06.229694 18712 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.229720 18610 ts_tablet_manager.cc:1367] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: Time spent bootstrapping tablet: real 0.383s user 0.029s sys 0.000s
I20240502 01:59:06.229887 18517 tablet_bootstrap.cc:492] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:06.230089 18517 ts_tablet_manager.cc:1367] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.279s user 0.030s sys 0.000s
I20240502 01:59:06.230731 18517 raft_consensus.cc:348] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.230895 18517 raft_consensus.cc:729] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:06.231017 18517 consensus_queue.cc:260] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.230268 18610 raft_consensus.cc:348] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.231434 18610 raft_consensus.cc:729] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d4e5422f66454511ae5a50ec6ab134ad, State: Initialized, Role: LEARNER
I20240502 01:59:06.231600 18610 consensus_queue.cc:260] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 1.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.232009 18610 ts_tablet_manager.cc:1398] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:06.232643 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session d4e5422f66454511ae5a50ec6ab134ad-e2f4ca025890492880608c7975650969 received from {username='slave'} at 127.7.221.70:33955
I20240502 01:59:06.232738 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session d4e5422f66454511ae5a50ec6ab134ad-e2f4ca025890492880608c7975650969 on tablet e2f4ca025890492880608c7975650969 with peer d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:06.234019 15197 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.234741 18071 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.235087 18590 ts_tablet_manager.cc:1398] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7: Time spent starting tablet: real 0.023s user 0.001s sys 0.000s
I20240502 01:59:06.235605 15197 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.235864 18517 ts_tablet_manager.cc:1398] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.006s user 0.001s sys 0.000s
I20240502 01:59:06.236795 15488 tablet_copy_service.cc:342] P 7b66a8940fe54e21bec1b2db25b2fed5: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-fcfc8dc4b90a4a6aa7e4035bbfdb4532 received from {username='slave'} at 127.7.221.69:53377
I20240502 01:59:06.236876 15488 tablet_copy_service.cc:433] P 7b66a8940fe54e21bec1b2db25b2fed5: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-fcfc8dc4b90a4a6aa7e4035bbfdb4532 on tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 with peer 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:06.236927 18654 log.cc:924] Time spent T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8: Append to log took a long time: real 0.086s user 0.004s sys 0.000s
I20240502 01:59:06.237120 15487 tablet_copy_service.cc:342] P 7b66a8940fe54e21bec1b2db25b2fed5: Request end of tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-ac5ec2c673cb4b988d1099e0442ec63f received from {username='slave'} at 127.7.221.72:51521
I20240502 01:59:06.237182 15487 tablet_copy_service.cc:433] P 7b66a8940fe54e21bec1b2db25b2fed5: ending tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-ac5ec2c673cb4b988d1099e0442ec63f on tablet ac5ec2c673cb4b988d1099e0442ec63f with peer 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:06.237193 18654 tablet_bootstrap.cc:492] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.237241 15197 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.237424 18654 tablet_bootstrap.cc:492] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap complete.
I20240502 01:59:06.237598 18654 ts_tablet_manager.cc:1367] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8: Time spent bootstrapping tablet: real 0.403s user 0.020s sys 0.008s
I20240502 01:59:06.238102 18654 raft_consensus.cc:348] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.238248 18654 raft_consensus.cc:729] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fc067ee690bb457e8109ad8e4eb43ab8, State: Initialized, Role: LEARNER
I20240502 01:59:06.239635 17244 raft_consensus.cc:1203] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.5->[1.6-1.6] Dedup: 1.6->[]
I20240502 01:59:06.241597 18654 consensus_queue.cc:260] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 1.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.242213 18654 ts_tablet_manager.cc:1398] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8: Time spent starting tablet: real 0.005s user 0.001s sys 0.000s
I20240502 01:59:06.242738 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-856f623374824d94913db0cb9faa25d1 received from {username='slave'} at 127.7.221.71:39719
I20240502 01:59:06.242815 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-856f623374824d94913db0cb9faa25d1 on tablet 856f623374824d94913db0cb9faa25d1 with peer fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:06.243433 18367 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.244722 16838 raft_consensus.cc:1203] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:06.245743 15331 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.246642 14915 catalog_manager.cc:5484] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:06.247262 15061 consensus_queue.cc:237] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.248822 15465 raft_consensus.cc:1261] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.249177 15331 raft_consensus.cc:1261] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.249224 18319 consensus_queue.cc:1035] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.249274 17106 raft_consensus.cc:1261] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 01:59:06.249958 14995 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 53edbd87d0d5471ca7626ca2071f75eb. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.250034 16840 raft_consensus.cc:1203] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 01:59:06.250643 18386 consensus_queue.cc:1035] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.250910 18386 consensus_queue.cc:1035] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.251365 18386 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.253459 15465 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.254222 14915 catalog_manager.cc:5484] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:06.258193 18151 raft_consensus.cc:1050] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09: attempting to promote NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 to VOTER
I20240502 01:59:06.258623 18151 consensus_queue.cc:237] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.258774 17106 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.260195 15465 raft_consensus.cc:1261] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
W20240502 01:59:06.260825 14995 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 53edbd87d0d5471ca7626ca2071f75eb. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.261243 18311 consensus_queue.cc:1035] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.261987 15330 raft_consensus.cc:1261] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.262027 17106 raft_consensus.cc:1261] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.262441 18311 consensus_queue.cc:1035] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.262996 18311 consensus_queue.cc:1035] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.264400 15330 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.266804 18319 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.268512 17106 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.268770 15331 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.270502 15465 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.271230 14913 catalog_manager.cc:5484] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.276639 15060 consensus_queue.cc:237] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:06.277863 17106 raft_consensus.cc:1261] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.278141 15465 raft_consensus.cc:1261] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.278199 15331 raft_consensus.cc:1261] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.278474 18624 consensus_queue.cc:1035] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.278609 18151 consensus_queue.cc:1035] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.278712 18386 consensus_queue.cc:1035] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.280242 18151 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.280663 17106 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.281785 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 53edbd87d0d5471ca7626ca2071f75eb with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.283077 15059 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.283581 15465 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.284593 15330 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.285997 16840 raft_consensus.cc:1203] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:06.286587 18517 ts_tablet_manager.cc:902] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:06.287091 18517 tablet_copy_client.cc:308] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:46123
I20240502 01:59:06.287284 14913 catalog_manager.cc:5484] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 6 to 7, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.289269 18745 tablet_bootstrap.cc:492] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.289992 16820 tablet_service.cc:1514] Processing DeleteTablet for tablet 53edbd87d0d5471ca7626ca2071f75eb with delete_type TABLET_DATA_TOMBSTONED (TS 84987ad9e4a54d96af8569e4aaf0ed64 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:55224
W20240502 01:59:06.290408 14902 catalog_manager.cc:4797] TS 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): delete failed for tablet 53edbd87d0d5471ca7626ca2071f75eb because the tablet was not found. No further retry: Not found: Tablet not found: 53edbd87d0d5471ca7626ca2071f75eb
I20240502 01:59:06.290467 15081 tablet_copy_service.cc:140] P ddae2a636423489693c12b7d7d166c09: Received BeginTabletCopySession request for tablet 77ae087802144c45924c1b79f561afe8 from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:51825)
I20240502 01:59:06.290561 15081 tablet_copy_service.cc:161] P ddae2a636423489693c12b7d7d166c09: Beginning new tablet copy session on tablet 77ae087802144c45924c1b79f561afe8 from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:51825: session id = 84987ad9e4a54d96af8569e4aaf0ed64-77ae087802144c45924c1b79f561afe8
I20240502 01:59:06.291390 18782 tablet_bootstrap.cc:492] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.291424 15081 tablet_copy_source_session.cc:205] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.291462 15059 consensus_queue.cc:237] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:06.291846 18782 tablet_bootstrap.cc:492] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap complete.
I20240502 01:59:06.292259 18517 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 77ae087802144c45924c1b79f561afe8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.293148 15465 raft_consensus.cc:1261] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.293607 18386 consensus_queue.cc:1035] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.294377 18517 tablet_copy_client.cc:783] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.294520 18517 tablet_copy_client.cc:647] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.292937 17106 raft_consensus.cc:1261] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.295188 18386 consensus_queue.cc:1035] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.295347 18745 tablet_bootstrap.cc:492] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap complete.
I20240502 01:59:06.295519 18745 ts_tablet_manager.cc:1367] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8: Time spent bootstrapping tablet: real 0.430s user 0.028s sys 0.000s
I20240502 01:59:06.296799 18386 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.298329 15464 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.298465 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 53edbd87d0d5471ca7626ca2071f75eb with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.297575 17106 raft_consensus.cc:2932] T 53edbd87d0d5471ca7626ca2071f75eb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.296393 18745 raft_consensus.cc:348] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.300406 18745 raft_consensus.cc:729] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fc067ee690bb457e8109ad8e4eb43ab8, State: Initialized, Role: LEARNER
I20240502 01:59:06.300522 18745 consensus_queue.cc:260] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.300930 14914 catalog_manager.cc:5484] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:06.303439 18782 ts_tablet_manager.cc:1367] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7: Time spent bootstrapping tablet: real 0.372s user 0.025s sys 0.000s
I20240502 01:59:06.303910 18782 raft_consensus.cc:348] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.304036 18782 raft_consensus.cc:729] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b8d615c71e44ff887e0f037c3efcff7, State: Initialized, Role: LEARNER
I20240502 01:59:06.304127 18782 consensus_queue.cc:260] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.304905 18782 ts_tablet_manager.cc:1398] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:06.294173 18712 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap complete.
I20240502 01:59:06.305228 18712 ts_tablet_manager.cc:1367] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: Time spent bootstrapping tablet: real 0.390s user 0.019s sys 0.008s
I20240502 01:59:06.305691 18712 raft_consensus.cc:348] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.305797 18712 raft_consensus.cc:729] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b8d615c71e44ff887e0f037c3efcff7, State: Initialized, Role: LEARNER
I20240502 01:59:06.305958 18712 consensus_queue.cc:260] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 1.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.306334 18712 ts_tablet_manager.cc:1398] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:06.307495 15081 tablet_copy_service.cc:342] P ddae2a636423489693c12b7d7d166c09: Request end of tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-1eb15147ff6c4d0b83b12dec736fc576 received from {username='slave'} at 127.7.221.72:45525
I20240502 01:59:06.307566 15081 tablet_copy_service.cc:433] P ddae2a636423489693c12b7d7d166c09: ending tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-1eb15147ff6c4d0b83b12dec736fc576 on tablet 1eb15147ff6c4d0b83b12dec736fc576 with peer 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:06.308132 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-e2f4ca025890492880608c7975650969 received from {username='slave'} at 127.7.221.72:47237
I20240502 01:59:06.308216 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-e2f4ca025890492880608c7975650969 on tablet e2f4ca025890492880608c7975650969 with peer 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:06.308146 15310 tablet_service.cc:1514] Processing DeleteTablet for tablet 53edbd87d0d5471ca7626ca2071f75eb with delete_type TABLET_DATA_TOMBSTONED (TS f77c2a969bde45b49ac1bbbe242ff171 not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:43766
I20240502 01:59:06.308641 18851 tablet_replica.cc:330] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171: stopping tablet replica
I20240502 01:59:06.308771 18851 raft_consensus.cc:2229] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:06.308948 18851 raft_consensus.cc:2258] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:06.309238 16971 raft_consensus.cc:1203] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:06.309831 18851 ts_tablet_manager.cc:1875] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.310279 18745 ts_tablet_manager.cc:1398] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8: Time spent starting tablet: real 0.014s user 0.001s sys 0.000s
I20240502 01:59:06.312580 15081 tablet_copy_service.cc:342] P ddae2a636423489693c12b7d7d166c09: Request end of tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-e00d1ed5cde044cb9bdebf787590261a received from {username='slave'} at 127.7.221.71:53189
I20240502 01:59:06.312654 15081 tablet_copy_service.cc:433] P ddae2a636423489693c12b7d7d166c09: ending tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-e00d1ed5cde044cb9bdebf787590261a on tablet e00d1ed5cde044cb9bdebf787590261a with peer fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:06.313575 18851 ts_tablet_manager.cc:1888] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:06.313639 18851 log.cc:1196] T 53edbd87d0d5471ca7626ca2071f75eb P f77c2a969bde45b49ac1bbbe242ff171: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/53edbd87d0d5471ca7626ca2071f75eb
I20240502 01:59:06.314100 14900 catalog_manager.cc:4830] TS f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): tablet 53edbd87d0d5471ca7626ca2071f75eb (table rebalance_test_table_1 [id=1c35c23ce11f4c249032c42d4c499e42]) successfully deleted
I20240502 01:59:06.317318 18517 tablet_copy_client.cc:523] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.318687 18517 tablet_bootstrap.cc:492] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
I20240502 01:59:06.322127 15465 consensus_queue.cc:237] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.323716 15059 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 01:59:06.324059 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3f938c5af51a43c9a23b753c4991e41d. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.324247 18767 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.324693 17244 raft_consensus.cc:1203] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
W20240502 01:59:06.325367 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3f938c5af51a43c9a23b753c4991e41d. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.325425 15197 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.327188 18767 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.328011 18767 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.329738 15197 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.331360 14913 catalog_manager.cc:5484] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:06.331946 18637 tablet_bootstrap.cc:492] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.332839 18637 tablet_bootstrap.cc:492] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:06.333012 18637 ts_tablet_manager.cc:1367] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.181s user 0.029s sys 0.000s
I20240502 01:59:06.333058 15197 consensus_queue.cc:237] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:06.333531 18637 raft_consensus.cc:348] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.333741 18637 raft_consensus.cc:729] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:06.333781 15059 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.333895 18637 consensus_queue.cc:260] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
W20240502 01:59:06.336125 15130 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cd00409d47954ee2b7555e7ff09a6e28. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.336153 18907 raft_consensus.cc:1050] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09: attempting to promote NON_VOTER d4e5422f66454511ae5a50ec6ab134ad to VOTER
I20240502 01:59:06.336611 18907 consensus_queue.cc:237] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:06.336764 16820 tablet_service.cc:1514] Processing DeleteTablet for tablet d1c832c721fb4b15bc6d266a1c295401 with delete_type TABLET_DATA_TOMBSTONED (Replica with old config index 8 (current committed config index is 9)) from {username='slave'} at 127.0.0.1:55224
W20240502 01:59:06.337253 14902 catalog_manager.cc:4810] TS 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): delete failed for tablet d1c832c721fb4b15bc6d266a1c295401 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet d1c832c721fb4b15bc6d266a1c295401 already in progress: copying tablet
W20240502 01:59:06.338141 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cd00409d47954ee2b7555e7ff09a6e28. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.338218 18818 tablet_bootstrap.cc:492] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.338567 18818 tablet_bootstrap.cc:492] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap complete.
I20240502 01:59:06.338845 15195 raft_consensus.cc:1261] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.338868 18818 ts_tablet_manager.cc:1367] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8: Time spent bootstrapping tablet: real 0.257s user 0.023s sys 0.005s
I20240502 01:59:06.339190 15465 raft_consensus.cc:1261] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.339238 18151 consensus_queue.cc:1035] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.339713 16971 raft_consensus.cc:1261] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.339746 18151 consensus_queue.cc:1035] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.340193 18151 consensus_queue.cc:1035] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.341380 15331 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.341821 17910 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.345028 18637 ts_tablet_manager.cc:1398] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.012s user 0.001s sys 0.000s
I20240502 01:59:06.347226 15081 tablet_copy_service.cc:342] P ddae2a636423489693c12b7d7d166c09: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-d1c832c721fb4b15bc6d266a1c295401 received from {username='slave'} at 127.7.221.69:51825
I20240502 01:59:06.347295 15081 tablet_copy_service.cc:433] P ddae2a636423489693c12b7d7d166c09: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-d1c832c721fb4b15bc6d266a1c295401 on tablet d1c832c721fb4b15bc6d266a1c295401 with peer 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:06.348685 18367 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.349016 15331 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.351560 18151 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.354502 14913 catalog_manager.cc:5484] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.356849 18818 raft_consensus.cc:348] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.356986 18818 raft_consensus.cc:729] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fc067ee690bb457e8109ad8e4eb43ab8, State: Initialized, Role: LEARNER
I20240502 01:59:06.357072 18782 ts_tablet_manager.cc:902] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:06.357090 18818 consensus_queue.cc:260] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.357403 18818 ts_tablet_manager.cc:1398] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8: Time spent starting tablet: real 0.018s user 0.000s sys 0.000s
I20240502 01:59:06.357205 15465 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.357512 18782 tablet_copy_client.cc:308] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:34323
I20240502 01:59:06.358420 15487 tablet_copy_service.cc:140] P 7b66a8940fe54e21bec1b2db25b2fed5: Received BeginTabletCopySession request for tablet 986e7c97e0734e958f364dc5d1167846 from peer 3b8d615c71e44ff887e0f037c3efcff7 ({username='slave'} at 127.7.221.72:51521)
I20240502 01:59:06.358510 15487 tablet_copy_service.cc:161] P 7b66a8940fe54e21bec1b2db25b2fed5: Beginning new tablet copy session on tablet 986e7c97e0734e958f364dc5d1167846 from peer 3b8d615c71e44ff887e0f037c3efcff7 at {username='slave'} at 127.7.221.72:51521: session id = 3b8d615c71e44ff887e0f037c3efcff7-986e7c97e0734e958f364dc5d1167846
I20240502 01:59:06.359431 15487 tablet_copy_source_session.cc:205] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.359732 15488 tablet_copy_service.cc:342] P 7b66a8940fe54e21bec1b2db25b2fed5: Request end of tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-6d5cffb7fcc54118861463e3b1fa4a8d received from {username='slave'} at 127.7.221.71:50737
I20240502 01:59:06.359788 15488 tablet_copy_service.cc:433] P 7b66a8940fe54e21bec1b2db25b2fed5: ending tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-6d5cffb7fcc54118861463e3b1fa4a8d on tablet 6d5cffb7fcc54118861463e3b1fa4a8d with peer fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:06.359834 14914 catalog_manager.cc:5484] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 4 to 5, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.360632 18737 tablet_bootstrap.cc:492] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.360634 18782 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 986e7c97e0734e958f364dc5d1167846. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.361382 15061 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.362377 18737 tablet_bootstrap.cc:492] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap complete.
I20240502 01:59:06.362263 15197 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.362610 18737 ts_tablet_manager.cc:1367] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8: Time spent bootstrapping tablet: real 0.307s user 0.023s sys 0.005s
I20240502 01:59:06.363045 18737 raft_consensus.cc:348] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.363145 18737 raft_consensus.cc:729] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fc067ee690bb457e8109ad8e4eb43ab8, State: Initialized, Role: LEARNER
I20240502 01:59:06.363242 18737 consensus_queue.cc:260] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.363513 18737 ts_tablet_manager.cc:1398] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:06.364408 16820 tablet_service.cc:1514] Processing DeleteTablet for tablet d1c832c721fb4b15bc6d266a1c295401 with delete_type TABLET_DATA_TOMBSTONED (Replica with old config index 8 (current committed config index is 9)) from {username='slave'} at 127.0.0.1:55224
I20240502 01:59:06.365614 18862 tablet_replica.cc:330] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64: stopping tablet replica
I20240502 01:59:06.365969 18862 raft_consensus.cc:2229] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Raft consensus shutting down.
I20240502 01:59:06.366242 18862 raft_consensus.cc:2258] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Raft consensus is shut down!
I20240502 01:59:06.366719 18189 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.366824 15081 tablet_copy_service.cc:342] P ddae2a636423489693c12b7d7d166c09: Request end of tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-1eb15147ff6c4d0b83b12dec736fc576 received from {username='slave'} at 127.7.221.71:53189
I20240502 01:59:06.366897 15081 tablet_copy_service.cc:433] P ddae2a636423489693c12b7d7d166c09: ending tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-1eb15147ff6c4d0b83b12dec736fc576 on tablet 1eb15147ff6c4d0b83b12dec736fc576 with peer fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:06.368037 15059 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.370767 18862 ts_tablet_manager.cc:1875] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.372818 17244 raft_consensus.cc:1203] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:06.373060 18862 ts_tablet_manager.cc:1888] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 01:59:06.373142 18862 log.cc:1196] T d1c832c721fb4b15bc6d266a1c295401 P 84987ad9e4a54d96af8569e4aaf0ed64: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/wals/d1c832c721fb4b15bc6d266a1c295401
I20240502 01:59:06.373066 15195 consensus_queue.cc:237] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.373822 14902 catalog_manager.cc:4830] TS 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): tablet d1c832c721fb4b15bc6d266a1c295401 (table rebalance_test_table_5 [id=3bc69a0a1e474b9fac7330d68770bfbe]) successfully deleted
I20240502 01:59:06.374225 18782 tablet_copy_client.cc:783] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.374426 18782 tablet_copy_client.cc:647] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.375186 15465 raft_consensus.cc:1261] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:06.375798 15061 raft_consensus.cc:1261] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 01:59:06.375835 15131 consensus_peers.cc:485] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f97c5c9bb4c84957857efc76d1c7717f. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.376199 18189 consensus_queue.cc:1035] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:06.376976 18189 consensus_queue.cc:1035] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:06.378970 17910 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.380052 15465 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.382527 14913 catalog_manager.cc:5484] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.383265 18517 tablet_bootstrap.cc:492] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.383548 18517 tablet_bootstrap.cc:492] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:06.383762 18517 ts_tablet_manager.cc:1367] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.065s user 0.024s sys 0.000s
I20240502 01:59:06.384181 18517 raft_consensus.cc:348] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.384297 18517 raft_consensus.cc:729] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:06.384444 18517 consensus_queue.cc:260] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.386090 18517 ts_tablet_manager.cc:1398] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 01:59:06.387734 15081 tablet_copy_service.cc:342] P ddae2a636423489693c12b7d7d166c09: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-77ae087802144c45924c1b79f561afe8 received from {username='slave'} at 127.7.221.69:51825
I20240502 01:59:06.387832 15081 tablet_copy_service.cc:433] P ddae2a636423489693c12b7d7d166c09: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-77ae087802144c45924c1b79f561afe8 on tablet 77ae087802144c45924c1b79f561afe8 with peer 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:06.393519 15061 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.397635 15465 consensus_queue.cc:237] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:06.399453 17106 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.400003 16838 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.400209 15061 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.400357 16840 raft_consensus.cc:1203] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:06.400763 18279 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.401027 18767 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.401585 15197 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
W20240502 01:59:06.401664 15398 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fcfc8dc4b90a4a6aa7e4035bbfdb4532. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.401793 18767 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.403548 18782 tablet_copy_client.cc:523] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.403841 16971 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.405102 18782 tablet_bootstrap.cc:492] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap starting.
I20240502 01:59:06.407781 18110 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.408244 18110 raft_consensus.cc:1013] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: attempt to promote peer fc067ee690bb457e8109ad8e4eb43ab8: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:06.409421 15059 consensus_queue.cc:237] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:06.409869 18767 raft_consensus.cc:1013] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: attempt to promote peer 84987ad9e4a54d96af8569e4aaf0ed64: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:06.410692 15195 raft_consensus.cc:1261] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.410728 16971 raft_consensus.cc:1261] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.411109 18150 consensus_queue.cc:1035] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.411283 18319 consensus_queue.cc:1035] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.412906 18319 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.414731 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 8f2fb504a49c44edac92fd935555c278 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.415307 15195 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.415967 16971 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.417349 18279 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.417717 14913 catalog_manager.cc:5484] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 5 to 6, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:06.420555 18189 raft_consensus.cc:1050] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER d4e5422f66454511ae5a50ec6ab134ad to VOTER
I20240502 01:59:06.421042 18189 consensus_queue.cc:237] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:06.422228 16840 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.424015 17106 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.425346 17244 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.425833 18109 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.427129 14909 catalog_manager.cc:5484] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 reported cstate change: config changed from index 5 to 6, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.427302 15061 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.429128 15331 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.429519 17108 raft_consensus.cc:1203] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:06.429677 17910 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.430399 18766 raft_consensus.cc:1050] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 to VOTER
I20240502 01:59:06.430922 18766 consensus_queue.cc:237] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:06.435679 15464 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.435704 15446 tablet_service.cc:1514] Processing DeleteTablet for tablet 8f2fb504a49c44edac92fd935555c278 with delete_type TABLET_DATA_TOMBSTONED (TS 7b66a8940fe54e21bec1b2db25b2fed5 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:60890
W20240502 01:59:06.436009 15398 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fcfc8dc4b90a4a6aa7e4035bbfdb4532. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.437815 18517 ts_tablet_manager.cc:902] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:06.438267 18517 tablet_copy_client.cc:308] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:06.439136 17108 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.439131 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet 6c4e85e7050b4dc8a4ce6bba818854a2 from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:43177)
I20240502 01:59:06.439229 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet 6c4e85e7050b4dc8a4ce6bba818854a2 from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:43177: session id = 84987ad9e4a54d96af8569e4aaf0ed64-6c4e85e7050b4dc8a4ce6bba818854a2
I20240502 01:59:06.440117 15216 tablet_copy_source_session.cc:205] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.440829 16971 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.441201 17910 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.443030 18517 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6c4e85e7050b4dc8a4ce6bba818854a2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.443848 18999 tablet_replica.cc:330] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5: stopping tablet replica
I20240502 01:59:06.444190 18999 raft_consensus.cc:2229] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:06.444463 18999 raft_consensus.cc:2258] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:06.445387 18517 tablet_copy_client.cc:783] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.445549 18517 tablet_copy_client.cc:647] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.447355 17307 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.448160 17910 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.450206 18999 ts_tablet_manager.cc:1875] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.451267 18782 tablet_bootstrap.cc:492] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.451493 15061 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.451685 18782 tablet_bootstrap.cc:492] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap complete.
I20240502 01:59:06.451884 18782 ts_tablet_manager.cc:1367] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7: Time spent bootstrapping tablet: real 0.047s user 0.026s sys 0.000s
I20240502 01:59:06.451999 17307 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.452522 18782 raft_consensus.cc:348] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.452702 18782 raft_consensus.cc:729] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b8d615c71e44ff887e0f037c3efcff7, State: Initialized, Role: LEARNER
I20240502 01:59:06.452837 18782 consensus_queue.cc:260] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.453356 18818 ts_tablet_manager.cc:902] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:06.453732 18782 ts_tablet_manager.cc:1398] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:06.453723 18999 ts_tablet_manager.cc:1888] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 01:59:06.453814 18999 log.cc:1196] T 8f2fb504a49c44edac92fd935555c278 P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/wals/8f2fb504a49c44edac92fd935555c278
I20240502 01:59:06.453853 18818 tablet_copy_client.cc:308] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:06.454321 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet 6c4e85e7050b4dc8a4ce6bba818854a2 from peer fc067ee690bb457e8109ad8e4eb43ab8 ({username='slave'} at 127.7.221.71:39719)
I20240502 01:59:06.454389 14901 catalog_manager.cc:4830] TS 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): tablet 8f2fb504a49c44edac92fd935555c278 (table rebalance_test_table_6 [id=2739aac0519d438587e8ec91530f5579]) successfully deleted
I20240502 01:59:06.454423 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet 6c4e85e7050b4dc8a4ce6bba818854a2 from peer fc067ee690bb457e8109ad8e4eb43ab8 at {username='slave'} at 127.7.221.71:39719: session id = fc067ee690bb457e8109ad8e4eb43ab8-6c4e85e7050b4dc8a4ce6bba818854a2
I20240502 01:59:06.454779 15488 tablet_copy_service.cc:342] P 7b66a8940fe54e21bec1b2db25b2fed5: Request end of tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-986e7c97e0734e958f364dc5d1167846 received from {username='slave'} at 127.7.221.72:51521
I20240502 01:59:06.454854 15488 tablet_copy_service.cc:433] P 7b66a8940fe54e21bec1b2db25b2fed5: ending tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-986e7c97e0734e958f364dc5d1167846 on tablet 986e7c97e0734e958f364dc5d1167846 with peer 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:06.455440 15216 tablet_copy_source_session.cc:205] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.456339 18818 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6c4e85e7050b4dc8a4ce6bba818854a2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.458791 18818 tablet_copy_client.cc:783] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.458941 18818 tablet_copy_client.cc:647] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.459451 16840 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.459848 17307 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.460503 17307 raft_consensus.cc:1050] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER d4e5422f66454511ae5a50ec6ab134ad to VOTER
I20240502 01:59:06.460906 17307 consensus_queue.cc:237] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:06.461658 15465 consensus_queue.cc:237] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.462476 17242 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.463014 15331 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.463388 15197 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.463505 18110 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.463747 18767 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.463749 18110 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.465564 15059 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.465564 16972 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.465853 18767 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.466014 15060 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
W20240502 01:59:06.466081 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3f938c5af51a43c9a23b753c4991e41d. This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:06.466210 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3f938c5af51a43c9a23b753c4991e41d. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.466562 17243 raft_consensus.cc:1203] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:06.467774 15197 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.469177 18296 raft_consensus.cc:1013] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: attempt to promote peer 3b8d615c71e44ff887e0f037c3efcff7: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:06.469708 18110 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20240502 01:59:06.469830 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3f938c5af51a43c9a23b753c4991e41d. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.469951 18110 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.470163 18296 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.470878 14914 catalog_manager.cc:5484] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 5 to 6, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.471261 15059 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.473970 15197 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.474432 18279 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.475144 18767 raft_consensus.cc:1013] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: attempt to promote peer 84987ad9e4a54d96af8569e4aaf0ed64: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:06.475162 15197 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.478356 17307 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.478747 15197 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.481367 17106 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.482226 14915 catalog_manager.cc:5484] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.483526 18109 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.484057 15464 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.484748 16840 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.485556 15330 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.489450 17910 raft_consensus.cc:1050] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 to VOTER
I20240502 01:59:06.489840 17910 consensus_queue.cc:237] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:06.490217 18110 raft_consensus.cc:1050] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
I20240502 01:59:06.490608 18110 consensus_queue.cc:237] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:06.492116 18818 tablet_copy_client.cc:523] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.492309 17244 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
W20240502 01:59:06.492991 15398 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fcfc8dc4b90a4a6aa7e4035bbfdb4532. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.493818 18818 tablet_bootstrap.cc:492] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap starting.
I20240502 01:59:06.493891 17108 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.494002 17242 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.494688 18296 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.494786 15464 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.495110 15331 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.495322 18189 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.496054 18880 raft_consensus.cc:1050] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
I20240502 01:59:06.496466 18880 consensus_queue.cc:237] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:06.498157 16838 raft_consensus.cc:1261] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.498157 14908 catalog_manager.cc:5484] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.498576 18417 consensus_queue.cc:1035] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.499980 15058 raft_consensus.cc:1261] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.501605 18829 consensus_queue.cc:1035] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.501642 18367 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.501665 16840 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.502099 18782 ts_tablet_manager.cc:902] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:06.502243 15197 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.502526 18782 tablet_copy_client.cc:308] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 01:59:06.503350 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet ce85785e670a4264b44797bb27cb4ecb from peer 3b8d615c71e44ff887e0f037c3efcff7 ({username='slave'} at 127.7.221.72:47237)
I20240502 01:59:06.503446 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet ce85785e670a4264b44797bb27cb4ecb from peer 3b8d615c71e44ff887e0f037c3efcff7 at {username='slave'} at 127.7.221.72:47237: session id = 3b8d615c71e44ff887e0f037c3efcff7-ce85785e670a4264b44797bb27cb4ecb
I20240502 01:59:06.504230 15351 tablet_copy_source_session.cc:205] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.504527 18367 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.504997 18782 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ce85785e670a4264b44797bb27cb4ecb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.506465 17242 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.507421 15331 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.507944 14913 catalog_manager.cc:5484] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:06.510125 18279 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.510396 15464 raft_consensus.cc:1261] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.510419 18279 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.510740 18109 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.510996 18782 tablet_copy_client.cc:783] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.511138 18782 tablet_copy_client.cc:647] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.511806 15464 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.512071 18279 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.514030 18829 consensus_queue.cc:1035] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.514147 16839 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.517166 15197 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.518090 18782 tablet_copy_client.cc:523] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.518815 17106 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:06.519992 15464 consensus_queue.cc:237] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.520059 14914 catalog_manager.cc:5484] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 7 to 8, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:06.522043 17242 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.522444 18296 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20240502 01:59:06.522886 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ac5ec2c673cb4b988d1099e0442ec63f. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.524000 15330 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.524518 15197 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.525030 18766 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.525138 18110 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.525719 15464 consensus_queue.cc:237] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:06.525856 17106 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.526243 18766 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.527729 17108 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.528251 18767 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.528445 15195 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.529531 18767 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.530503 16840 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.530990 18279 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.531086 16972 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.531318 18782 tablet_bootstrap.cc:492] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap starting.
I20240502 01:59:06.532505 18109 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.533288 16972 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.534111 17307 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.534889 16972 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.536314 18296 raft_consensus.cc:1013] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: attempt to promote peer fc067ee690bb457e8109ad8e4eb43ab8: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:06.536841 16840 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.539673 18517 tablet_copy_client.cc:523] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.540907 18517 tablet_bootstrap.cc:492] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
I20240502 01:59:06.544471 18296 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.547433 18417 raft_consensus.cc:2932] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.547605 15197 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.547443 15330 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.550072 14909 catalog_manager.cc:5484] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 reported cstate change: config changed from index 8 to 9, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.550899 16838 raft_consensus.cc:2932] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.551751 14908 catalog_manager.cc:5484] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 5 to 6, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.552906 17106 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.555439 15058 raft_consensus.cc:2932] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.556291 16954 tablet_service.cc:1514] Processing DeleteTablet for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 with delete_type TABLET_DATA_TOMBSTONED (TS d4e5422f66454511ae5a50ec6ab134ad not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:52144
I20240502 01:59:06.557201 18737 ts_tablet_manager.cc:902] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:06.557336 14909 catalog_manager.cc:5484] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64 reported cstate change: config changed from index 4 to 5, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.557762 18737 tablet_copy_client.cc:308] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
W20240502 01:59:06.558286 14901 catalog_manager.cc:4797] TS d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): delete failed for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 because the tablet was not found. No further retry: Not found: Tablet not found: fcfc8dc4b90a4a6aa7e4035bbfdb4532
I20240502 01:59:06.559170 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.559172 17108 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.560245 16972 raft_consensus.cc:1203] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 01:59:06.561098 17105 raft_consensus.cc:1203] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:06.562027 15197 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.563593 18296 raft_consensus.cc:1050] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 to VOTER
I20240502 01:59:06.564047 18296 consensus_queue.cc:237] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.565761 18296 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.568107 17105 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.568109 15330 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.570071 18839 raft_consensus.cc:1050] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
I20240502 01:59:06.570538 18839 consensus_queue.cc:237] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.572013 19072 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.572285 19072 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.572470 16838 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.572662 15330 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.572777 18367 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.574266 15060 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.574739 15197 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.575034 17106 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
W20240502 01:59:06.575383 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ac5ec2c673cb4b988d1099e0442ec63f. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.575374 18367 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.575819 17243 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.577054 18637 ts_tablet_manager.cc:902] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:06.577481 18637 tablet_copy_client.cc:308] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 01:59:06.577615 15465 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.578096 15060 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.578852 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet 091d27d51acc4db0a1c7d20deeff5c10 from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:51343)
I20240502 01:59:06.578972 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet 091d27d51acc4db0a1c7d20deeff5c10 from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:51343: session id = 84987ad9e4a54d96af8569e4aaf0ed64-091d27d51acc4db0a1c7d20deeff5c10
I20240502 01:59:06.579103 18766 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.579432 18766 raft_consensus.cc:1050] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 to VOTER
I20240502 01:59:06.580003 15351 tablet_copy_source_session.cc:205] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.579981 18766 consensus_queue.cc:237] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:06.581557 14915 catalog_manager.cc:5484] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.582274 18189 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.582754 18637 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 091d27d51acc4db0a1c7d20deeff5c10. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.582549 18189 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.583914 15058 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.584324 18767 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.585348 15331 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.585358 18637 tablet_copy_client.cc:783] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.585523 18637 tablet_copy_client.cc:647] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.586086 17243 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.588981 18110 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.589624 15464 raft_consensus.cc:2932] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.591646 18637 tablet_copy_client.cc:523] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.593021 18637 tablet_bootstrap.cc:492] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
I20240502 01:59:06.593730 18767 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.594992 18767 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.595261 16971 raft_consensus.cc:1203] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Deduplicated request from leader. Original: 1.5->[1.6-1.6] Dedup: 1.6->[]
I20240502 01:59:06.595315 15331 consensus_queue.cc:237] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:06.597110 15058 raft_consensus.cc:1261] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.597671 15330 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.597949 18071 consensus_queue.cc:1035] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.599186 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:06.599648 17243 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.599911 18071 raft_consensus.cc:2932] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.600761 16838 raft_consensus.cc:1261] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.601389 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9c2892024ce942e1bf1834c1b9ad2049 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.601454 18880 consensus_queue.cc:1035] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.602280 14909 catalog_manager.cc:5484] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7 reported cstate change: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.602392 16838 raft_consensus.cc:2932] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.603871 15058 raft_consensus.cc:2932] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.604545 18110 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.607147 15056 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.607314 15330 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.608494 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet ce85785e670a4264b44797bb27cb4ecb from peer fc067ee690bb457e8109ad8e4eb43ab8 ({username='slave'} at 127.7.221.71:41349)
I20240502 01:59:06.608604 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet ce85785e670a4264b44797bb27cb4ecb from peer fc067ee690bb457e8109ad8e4eb43ab8 at {username='slave'} at 127.7.221.71:41349: session id = fc067ee690bb457e8109ad8e4eb43ab8-ce85785e670a4264b44797bb27cb4ecb
I20240502 01:59:06.609568 15351 tablet_copy_source_session.cc:205] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.609650 15197 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.611292 18737 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ce85785e670a4264b44797bb27cb4ecb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.611759 17108 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.613278 14914 catalog_manager.cc:5484] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 5 to 6, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.613847 18367 raft_consensus.cc:1050] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
I20240502 01:59:06.614041 18737 tablet_copy_client.cc:783] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.614197 18737 tablet_copy_client.cc:647] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.614250 18367 consensus_queue.cc:237] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.615362 14909 catalog_manager.cc:5484] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.616883 16839 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.617276 18839 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.617904 18737 tablet_copy_client.cc:523] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.618008 15464 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.619077 18737 tablet_bootstrap.cc:492] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap starting.
I20240502 01:59:06.619056 18839 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.620276 17108 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.620580 18839 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.623860 15446 tablet_service.cc:1514] Processing DeleteTablet for tablet 9c2892024ce942e1bf1834c1b9ad2049 with delete_type TABLET_DATA_TOMBSTONED (TS 7b66a8940fe54e21bec1b2db25b2fed5 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:60890
I20240502 01:59:06.624047 18999 tablet_replica.cc:330] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5: stopping tablet replica
I20240502 01:59:06.624162 18999 raft_consensus.cc:2229] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:06.624282 18999 raft_consensus.cc:2258] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:06.625108 18999 ts_tablet_manager.cc:1875] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.625211 17243 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.625583 18839 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.626328 15330 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.626721 18839 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.627329 18999 ts_tablet_manager.cc:1888] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 01:59:06.627411 18999 log.cc:1196] T 9c2892024ce942e1bf1834c1b9ad2049 P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/wals/9c2892024ce942e1bf1834c1b9ad2049
I20240502 01:59:06.628878 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:06.629341 14901 catalog_manager.cc:4830] TS 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): tablet 9c2892024ce942e1bf1834c1b9ad2049 (table rebalance_test_table_8 [id=bf7d5082701149bea6b9e5c480f53c10]) successfully deleted
I20240502 01:59:06.632375 15446 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:06.633658 18839 raft_consensus.cc:1013] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer fc067ee690bb457e8109ad8e4eb43ab8: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:06.634802 18829 raft_consensus.cc:1050] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
I20240502 01:59:06.634837 18150 raft_consensus.cc:1050] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
I20240502 01:59:06.635210 18150 consensus_queue.cc:237] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:06.635280 18829 consensus_queue.cc:237] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.637094 15195 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.637256 16837 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.637481 18880 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.637723 18880 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.637807 16971 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.638437 15193 raft_consensus.cc:1261] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.638813 18319 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.640851 17105 raft_consensus.cc:1203] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:06.641956 15056 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.642913 16838 raft_consensus.cc:1261] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.643283 18319 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.643469 15466 consensus_queue.cc:237] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:06.644630 18071 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.644845 18880 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.646250 17243 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.646670 18880 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.650933 15330 raft_consensus.cc:1261] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.651667 18150 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.653156 17105 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:06.653416 15466 consensus_queue.cc:237] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.653489 16837 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:06.653558 18766 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:06.654383 15463 consensus_queue.cc:237] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:06.654783 18818 tablet_bootstrap.cc:492] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.654922 18766 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:06.655079 18818 tablet_bootstrap.cc:492] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap complete.
I20240502 01:59:06.655277 18818 ts_tablet_manager.cc:1367] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: Time spent bootstrapping tablet: real 0.162s user 0.027s sys 0.000s
I20240502 01:59:06.655570 15193 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:06.655815 18818 raft_consensus.cc:348] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.655865 17105 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.656253 18818 raft_consensus.cc:729] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fc067ee690bb457e8109ad8e4eb43ab8, State: Initialized, Role: LEARNER
I20240502 01:59:06.656381 18818 consensus_queue.cc:260] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.657182 18110 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.657451 17240 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 01:59:06.657783 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ac5ec2c673cb4b988d1099e0442ec63f. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.658030 15329 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.658218 18818 ts_tablet_manager.cc:1398] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 01:59:06.658335 15328 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.658939 18279 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:06.659145 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-6c4e85e7050b4dc8a4ce6bba818854a2 received from {username='slave'} at 127.7.221.71:39719
I20240502 01:59:06.659220 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-6c4e85e7050b4dc8a4ce6bba818854a2 on tablet 6c4e85e7050b4dc8a4ce6bba818854a2 with peer fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:06.659257 18279 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.659479 18279 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.659658 18279 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.660288 18279 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.660379 17105 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.664732 15058 consensus_queue.cc:237] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:06.665442 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ac5ec2c673cb4b988d1099e0442ec63f with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.665829 15193 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.666321 16970 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:06.666371 15197 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:06.666842 18319 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:06.667475 18319 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:06.669559 18767 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.670469 18150 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.671422 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:06.671384 15328 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.671516 15193 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.672040 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d1c832c721fb4b15bc6d266a1c295401 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.672252 16970 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.672938 14915 catalog_manager.cc:5484] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 7 to 8, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:06.674115 15329 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.674717 14913 catalog_manager.cc:5484] T d1c832c721fb4b15bc6d266a1c295401 P d4e5422f66454511ae5a50ec6ab134ad reported cstate change: config changed from index 9 to 10, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.676432 14914 catalog_manager.cc:5484] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 9 to 10, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.676505 14906 catalog_manager.cc:5484] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.683388 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:06.684827 18610 ts_tablet_manager.cc:902] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:06.685258 18610 tablet_copy_client.cc:308] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 01:59:06.685643 18189 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.686738 15310 tablet_service.cc:1514] Processing DeleteTablet for tablet d1c832c721fb4b15bc6d266a1c295401 with delete_type TABLET_DATA_TOMBSTONED (TS f77c2a969bde45b49ac1bbbe242ff171 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:43766
I20240502 01:59:06.686960 15309 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:06.687139 18851 tablet_replica.cc:330] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171: stopping tablet replica
I20240502 01:59:06.687284 18851 raft_consensus.cc:2229] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:06.687501 18851 raft_consensus.cc:2258] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:06.688462 18851 ts_tablet_manager.cc:1875] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.688378 16837 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.688499 15060 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.690526 18851 ts_tablet_manager.cc:1888] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 01:59:06.691591 17105 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.693260 17471 raft_consensus.cc:1050] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 to VOTER
I20240502 01:59:06.693598 17471 consensus_queue.cc:237] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:06.695698 15060 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.695856 16837 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.696457 18839 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.696738 18839 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.696825 14909 catalog_manager.cc:5484] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 reported cstate change: config changed from index 5 to 6, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.698952 18110 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.699040 16836 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.701073 17105 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.690582 18851 log.cc:1196] T d1c832c721fb4b15bc6d266a1c295401 P f77c2a969bde45b49ac1bbbe242ff171: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/d1c832c721fb4b15bc6d266a1c295401
I20240502 01:59:06.701567 17105 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.702661 18189 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.703198 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.703826 15041 tablet_service.cc:1514] Processing DeleteTablet for tablet fc07e430e3e74e3885f263881bd04976 with delete_type TABLET_DATA_TOMBSTONED (TS ddae2a636423489693c12b7d7d166c09 not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:53384
I20240502 01:59:06.704906 14900 catalog_manager.cc:4830] TS f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): tablet d1c832c721fb4b15bc6d266a1c295401 (table rebalance_test_table_5 [id=3bc69a0a1e474b9fac7330d68770bfbe]) successfully deleted
I20240502 01:59:06.705137 19132 tablet_replica.cc:330] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09: stopping tablet replica
I20240502 01:59:06.705613 15174 tablet_service.cc:1514] Processing DeleteTablet for tablet ac5ec2c673cb4b988d1099e0442ec63f with delete_type TABLET_DATA_TOMBSTONED (TS 26a3d5ced97d4dcc919b80fde53f6a5b not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:37206
I20240502 01:59:06.705749 19132 raft_consensus.cc:2229] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:06.706053 19132 raft_consensus.cc:2258] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:06.706723 15040 tablet_service.cc:1514] Processing DeleteTablet for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 with delete_type TABLET_DATA_TOMBSTONED (TS ddae2a636423489693c12b7d7d166c09 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:53384
I20240502 01:59:06.691056 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet 091d27d51acc4db0a1c7d20deeff5c10 from peer d4e5422f66454511ae5a50ec6ab134ad ({username='slave'} at 127.7.221.70:33955)
I20240502 01:59:06.707026 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet 091d27d51acc4db0a1c7d20deeff5c10 from peer d4e5422f66454511ae5a50ec6ab134ad at {username='slave'} at 127.7.221.70:33955: session id = d4e5422f66454511ae5a50ec6ab134ad-091d27d51acc4db0a1c7d20deeff5c10
I20240502 01:59:06.707651 19132 ts_tablet_manager.cc:1875] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.707762 18839 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.708038 15351 tablet_copy_source_session.cc:205] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.708890 18416 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.709638 17108 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.709863 18712 ts_tablet_manager.cc:902] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:06.710146 19132 ts_tablet_manager.cc:1888] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:06.710287 19132 log.cc:1196] T fc07e430e3e74e3885f263881bd04976 P ddae2a636423489693c12b7d7d166c09: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/fc07e430e3e74e3885f263881bd04976
I20240502 01:59:06.710330 18712 tablet_copy_client.cc:308] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 01:59:06.710906 14901 catalog_manager.cc:4830] TS ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123): tablet fc07e430e3e74e3885f263881bd04976 (table rebalance_test_table_1 [id=1c35c23ce11f4c249032c42d4c499e42]) successfully deleted
I20240502 01:59:06.711175 19132 tablet_replica.cc:330] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09: stopping tablet replica
I20240502 01:59:06.711369 19132 raft_consensus.cc:2229] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:06.711601 19132 pending_rounds.cc:62] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09: Trying to abort 1 pending ops.
I20240502 01:59:06.712054 19132 pending_rounds.cc:68] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09: Aborting op as it isn't in flight: id { term: 1 index: 7 } timestamp: 7023063639779655680 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "fcfc8dc4b90a4a6aa7e4035bbfdb4532" old_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } } new_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } } }
I20240502 01:59:06.712337 19132 raft_consensus.cc:2894] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Aborting config change with OpId 1.7: Aborted: Op aborted
I20240502 01:59:06.712245 14914 catalog_manager.cc:5484] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:06.712347 16836 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.712510 19132 raft_consensus.cc:2866] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 01:59:06.712571 19132 raft_consensus.cc:2258] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:06.712975 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet 30620f4774b8414eb4e27bd5ecb6ecee from peer 3b8d615c71e44ff887e0f037c3efcff7 ({username='slave'} at 127.7.221.72:47237)
I20240502 01:59:06.713060 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet 30620f4774b8414eb4e27bd5ecb6ecee from peer 3b8d615c71e44ff887e0f037c3efcff7 at {username='slave'} at 127.7.221.72:47237: session id = 3b8d615c71e44ff887e0f037c3efcff7-30620f4774b8414eb4e27bd5ecb6ecee
I20240502 01:59:06.713145 16837 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.713979 15351 tablet_copy_source_session.cc:205] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.714443 18610 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 091d27d51acc4db0a1c7d20deeff5c10. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.714841 18712 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 30620f4774b8414eb4e27bd5ecb6ecee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.714814 14906 catalog_manager.cc:5484] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.715001 15058 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.716962 18712 tablet_copy_client.cc:783] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.717123 18712 tablet_copy_client.cc:647] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.719501 18610 tablet_copy_client.cc:783] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.719643 18610 tablet_copy_client.cc:647] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.723076 17240 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.725781 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet fc07e430e3e74e3885f263881bd04976 with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.727953 15466 consensus_queue.cc:237] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:06.729295 15197 consensus_queue.cc:237] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:06.730187 18610 tablet_copy_client.cc:523] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.729831 15329 consensus_queue.cc:237] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.730547 15466 consensus_queue.cc:237] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:06.731866 18610 tablet_bootstrap.cc:492] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap starting.
I20240502 01:59:06.733130 17108 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.733621 18109 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.733831 17108 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:06.734438 16836 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:06.735271 16837 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.735566 18839 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.735586 17307 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:06.735891 17307 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:06.738706 17239 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.738844 15058 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.740149 18189 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.740558 17471 raft_consensus.cc:1013] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 3b8d615c71e44ff887e0f037c3efcff7: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:06.740521 17307 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.740633 17471 raft_consensus.cc:1013] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer fc067ee690bb457e8109ad8e4eb43ab8: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:06.741945 17105 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.742496 16836 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.742758 18880 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.744874 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d24986c2ebab46f086e3f1f5a53af26b with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.744783 17105 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.745640 14915 catalog_manager.cc:5484] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:06.745893 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:06.746923 17307 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.748090 16836 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.748996 17307 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:06.750278 15331 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.750943 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 with cas_config_opid_index 10: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.750813 17237 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:06.751083 14914 catalog_manager.cc:5484] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 10 to 11, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.751220 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet fc07e430e3e74e3885f263881bd04976 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.751492 17307 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.752558 15331 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:06.753523 14906 catalog_manager.cc:5484] T fc07e430e3e74e3885f263881bd04976 P 3b8d615c71e44ff887e0f037c3efcff7 reported cstate change: config changed from index 8 to 9, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:06.754348 15040 tablet_service.cc:1514] Processing DeleteTablet for tablet d24986c2ebab46f086e3f1f5a53af26b with delete_type TABLET_DATA_TOMBSTONED (TS ddae2a636423489693c12b7d7d166c09 not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:53384
I20240502 01:59:06.755512 15193 consensus_queue.cc:237] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:06.757845 16836 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.757863 17105 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.758038 16954 tablet_service.cc:1514] Processing DeleteTablet for tablet fc07e430e3e74e3885f263881bd04976 with delete_type TABLET_DATA_TOMBSTONED (TS d4e5422f66454511ae5a50ec6ab134ad not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:52144
I20240502 01:59:06.758265 18857 tablet_replica.cc:330] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: stopping tablet replica
I20240502 01:59:06.758314 17471 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.758535 17471 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.758562 18857 raft_consensus.cc:2229] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:06.758801 18857 pending_rounds.cc:62] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: Trying to abort 1 pending ops.
I20240502 01:59:06.758860 18857 pending_rounds.cc:68] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: Aborting op as it isn't in flight: id { term: 1 index: 6 } timestamp: 7023063639902760960 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "fc07e430e3e74e3885f263881bd04976" old_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } } new_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } } }
I20240502 01:59:06.759006 18857 raft_consensus.cc:2894] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Aborting config change with OpId 1.6: Aborted: Op aborted
I20240502 01:59:06.759083 18857 raft_consensus.cc:2866] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 01:59:06.759124 18857 raft_consensus.cc:2258] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Raft consensus is shut down!
I20240502 01:59:06.760416 17471 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.763072 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d24986c2ebab46f086e3f1f5a53af26b with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.763069 15174 tablet_service.cc:1514] Processing DeleteTablet for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 with delete_type TABLET_DATA_TOMBSTONED (TS 26a3d5ced97d4dcc919b80fde53f6a5b not found in new config with opid_index 11) from {username='slave'} at 127.0.0.1:37206
I20240502 01:59:06.763392 16837 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.764504 14914 catalog_manager.cc:5484] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:06.765995 16840 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.766672 15446 tablet_service.cc:1514] Processing DeleteTablet for tablet d24986c2ebab46f086e3f1f5a53af26b with delete_type TABLET_DATA_TOMBSTONED (TS 7b66a8940fe54e21bec1b2db25b2fed5 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:60890
I20240502 01:59:06.766867 18999 tablet_replica.cc:330] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: stopping tablet replica
I20240502 01:59:06.766795 17105 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.767011 18999 raft_consensus.cc:2229] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:06.767154 18999 pending_rounds.cc:62] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: Trying to abort 1 pending ops.
I20240502 01:59:06.767212 18999 pending_rounds.cc:68] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: Aborting op as it isn't in flight: id { term: 1 index: 6 } timestamp: 7023063640351834112 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "d24986c2ebab46f086e3f1f5a53af26b" old_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } } new_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } } }
I20240502 01:59:06.767344 18999 raft_consensus.cc:2894] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Aborting config change with OpId 1.6: Aborted: Op aborted
I20240502 01:59:06.767405 18999 raft_consensus.cc:2866] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 01:59:06.767452 18999 raft_consensus.cc:2258] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:06.766533 18880 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.001s
I20240502 01:59:06.769474 18880 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.771737 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e2f4ca025890492880608c7975650969 with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.772146 15058 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.772284 16837 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:06.774626 14915 catalog_manager.cc:5484] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 7 to 8, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 01:59:06.775411 18968 consensus_queue.cc:1159] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "ddae2a636423489693c12b7d7d166c09" status { last_received { term: 1 index: 7 } last_committed_idx: 6 last_received_current_leader { term: 1 index: 7 } }
I20240502 01:59:06.775605 18712 tablet_copy_client.cc:523] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.777362 18712 tablet_bootstrap.cc:492] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap starting.
I20240502 01:59:06.781380 19133 tablet_replica.cc:330] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b: stopping tablet replica
I20240502 01:59:06.781610 19133 raft_consensus.cc:2229] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:06.781895 19133 raft_consensus.cc:2258] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:06.782644 19133 ts_tablet_manager.cc:1875] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.784471 15331 consensus_queue.cc:237] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.785427 19133 ts_tablet_manager.cc:1888] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:06.785626 19133 log.cc:1196] T ac5ec2c673cb4b988d1099e0442ec63f P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/ac5ec2c673cb4b988d1099e0442ec63f
I20240502 01:59:06.785915 15060 raft_consensus.cc:1261] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:06.786621 18071 consensus_queue.cc:1035] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 01:59:06.786909 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6db29f2516e1486782be0312c9bcac8b. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.787135 19133 tablet_replica.cc:330] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b: stopping tablet replica
I20240502 01:59:06.787187 16954 tablet_service.cc:1514] Processing DeleteTablet for tablet e2f4ca025890492880608c7975650969 with delete_type TABLET_DATA_TOMBSTONED (TS d4e5422f66454511ae5a50ec6ab134ad not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:52144
I20240502 01:59:06.787390 19133 raft_consensus.cc:2229] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:06.787659 19133 raft_consensus.cc:2258] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:06.788062 15331 consensus_queue.cc:237] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:06.788166 15193 raft_consensus.cc:1261] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:06.788501 14900 catalog_manager.cc:4830] TS 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905): tablet ac5ec2c673cb4b988d1099e0442ec63f (table rebalance_test_table_5 [id=3bc69a0a1e474b9fac7330d68770bfbe]) successfully deleted
I20240502 01:59:06.788650 18071 consensus_queue.cc:1035] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:06.789304 16840 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.789805 15058 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.789784 18829 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.791143 18829 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.793196 19133 ts_tablet_manager.cc:1875] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.794087 18071 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.794420 16837 raft_consensus.cc:1203] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 01:59:06.795746 19133 ts_tablet_manager.cc:1888] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.10
I20240502 01:59:06.795879 19133 log.cc:1196] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/fcfc8dc4b90a4a6aa7e4035bbfdb4532
I20240502 01:59:06.795799 15061 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.797024 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e2f4ca025890492880608c7975650969 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.797362 16837 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.797785 14915 catalog_manager.cc:5484] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 8 to 9, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:06.799328 14900 catalog_manager.cc:4830] TS 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905): tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 (table rebalance_test_table_0 [id=9736e7c5365849759564e386d208932f]) successfully deleted
I20240502 01:59:06.799983 19132 ts_tablet_manager.cc:1875] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.801481 18782 tablet_bootstrap.cc:492] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.801723 18782 tablet_bootstrap.cc:492] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap complete.
I20240502 01:59:06.801885 18782 ts_tablet_manager.cc:1367] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7: Time spent bootstrapping tablet: real 0.271s user 0.021s sys 0.006s
I20240502 01:59:06.801973 19132 ts_tablet_manager.cc:1888] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:06.802029 19132 log.cc:1196] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P ddae2a636423489693c12b7d7d166c09: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/fcfc8dc4b90a4a6aa7e4035bbfdb4532
I20240502 01:59:06.802435 19132 tablet_replica.cc:330] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09: stopping tablet replica
I20240502 01:59:06.802371 18782 raft_consensus.cc:348] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.802484 18782 raft_consensus.cc:729] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b8d615c71e44ff887e0f037c3efcff7, State: Initialized, Role: LEARNER
I20240502 01:59:06.802552 19132 raft_consensus.cc:2229] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:06.803387 19158 ts_tablet_manager.cc:902] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:06.803938 19158 tablet_copy_client.cc:308] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:06.802572 18782 consensus_queue.cc:260] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:06.804335 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet cd00409d47954ee2b7555e7ff09a6e28 from peer d4e5422f66454511ae5a50ec6ab134ad ({username='slave'} at 127.7.221.70:59193)
I20240502 01:59:06.804458 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet cd00409d47954ee2b7555e7ff09a6e28 from peer d4e5422f66454511ae5a50ec6ab134ad at {username='slave'} at 127.7.221.70:59193: session id = d4e5422f66454511ae5a50ec6ab134ad-cd00409d47954ee2b7555e7ff09a6e28
I20240502 01:59:06.805313 15216 tablet_copy_source_session.cc:205] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.805984 18610 tablet_bootstrap.cc:492] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.806305 18782 ts_tablet_manager.cc:1398] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 01:59:06.806452 18610 tablet_bootstrap.cc:492] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap complete.
I20240502 01:59:06.806634 18610 ts_tablet_manager.cc:1367] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad: Time spent bootstrapping tablet: real 0.075s user 0.024s sys 0.000s
I20240502 01:59:06.806818 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-ce85785e670a4264b44797bb27cb4ecb received from {username='slave'} at 127.7.221.72:47237
I20240502 01:59:06.806902 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-ce85785e670a4264b44797bb27cb4ecb on tablet ce85785e670a4264b44797bb27cb4ecb with peer 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:06.807140 18610 raft_consensus.cc:348] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.807471 14901 catalog_manager.cc:4830] TS ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123): tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 (table rebalance_test_table_0 [id=9736e7c5365849759564e386d208932f]) successfully deleted
I20240502 01:59:06.807502 18610 raft_consensus.cc:729] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d4e5422f66454511ae5a50ec6ab134ad, State: Initialized, Role: LEARNER
I20240502 01:59:06.807698 18610 consensus_queue.cc:260] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.807943 18829 raft_consensus.cc:2932] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.808437 19158 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd00409d47954ee2b7555e7ff09a6e28. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.808604 18610 ts_tablet_manager.cc:1398] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 01:59:06.809900 15193 raft_consensus.cc:2932] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.810256 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session d4e5422f66454511ae5a50ec6ab134ad-091d27d51acc4db0a1c7d20deeff5c10 received from {username='slave'} at 127.7.221.70:33955
I20240502 01:59:06.810313 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session d4e5422f66454511ae5a50ec6ab134ad-091d27d51acc4db0a1c7d20deeff5c10 on tablet 091d27d51acc4db0a1c7d20deeff5c10 with peer d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:06.811097 19158 tablet_copy_client.cc:783] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.811249 19158 tablet_copy_client.cc:647] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.811098 15060 raft_consensus.cc:2932] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.813354 19132 pending_rounds.cc:62] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09: Trying to abort 1 pending ops.
I20240502 01:59:06.813403 19132 pending_rounds.cc:68] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09: Aborting op as it isn't in flight: id { term: 1 index: 7 } timestamp: 7023063640856104960 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "d24986c2ebab46f086e3f1f5a53af26b" old_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } } new_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } } }
I20240502 01:59:06.813520 19132 raft_consensus.cc:2894] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Aborting config change with OpId 1.7: Aborted: Op aborted
I20240502 01:59:06.813576 19132 raft_consensus.cc:2866] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 01:59:06.813624 19132 raft_consensus.cc:2258] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:06.814224 19132 ts_tablet_manager.cc:1875] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.814841 14906 catalog_manager.cc:5484] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index -1 to 4, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.817507 19132 ts_tablet_manager.cc:1888] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:06.817571 19132 log.cc:1196] T d24986c2ebab46f086e3f1f5a53af26b P ddae2a636423489693c12b7d7d166c09: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/d24986c2ebab46f086e3f1f5a53af26b
I20240502 01:59:06.818048 14901 catalog_manager.cc:4830] TS ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123): tablet d24986c2ebab46f086e3f1f5a53af26b (table rebalance_test_table_8 [id=bf7d5082701149bea6b9e5c480f53c10]) successfully deleted
I20240502 01:59:06.818936 17224 tablet_service.cc:1514] Processing DeleteTablet for tablet e2f4ca025890492880608c7975650969 with delete_type TABLET_DATA_TOMBSTONED (TS 3b8d615c71e44ff887e0f037c3efcff7 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:37116
I20240502 01:59:06.819597 15331 consensus_queue.cc:237] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:06.820164 19158 tablet_copy_client.cc:523] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.821266 16837 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:06.821450 19158 tablet_bootstrap.cc:492] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap starting.
I20240502 01:59:06.821672 18417 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:06.823042 15061 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:06.823473 18417 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.825484 16837 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.825686 14908 catalog_manager.cc:5484] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 9 to 10, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:06.826074 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e2f4ca025890492880608c7975650969 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.827054 18071 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:06.828081 19164 tablet_replica.cc:330] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: stopping tablet replica
I20240502 01:59:06.828223 19164 raft_consensus.cc:2229] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Raft consensus shutting down.
I20240502 01:59:06.828353 15061 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.828541 19164 pending_rounds.cc:62] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: Trying to abort 1 pending ops.
I20240502 01:59:06.828614 19164 pending_rounds.cc:68] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: Aborting op as it isn't in flight: id { term: 1 index: 7 } timestamp: 7023063640616800256 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "e2f4ca025890492880608c7975650969" old_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } } new_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } } }
I20240502 01:59:06.828730 17105 raft_consensus.cc:1203] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:06.828787 19164 raft_consensus.cc:2894] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Aborting config change with OpId 1.7: Aborted: Op aborted
I20240502 01:59:06.828856 19164 raft_consensus.cc:2866] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 01:59:06.828896 19164 raft_consensus.cc:2258] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Raft consensus is shut down!
I20240502 01:59:06.836804 15174 tablet_service.cc:1514] Processing DeleteTablet for tablet e2f4ca025890492880608c7975650969 with delete_type TABLET_DATA_TOMBSTONED (TS 26a3d5ced97d4dcc919b80fde53f6a5b not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:37206
I20240502 01:59:06.837051 19133 tablet_replica.cc:330] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: stopping tablet replica
I20240502 01:59:06.837209 19133 raft_consensus.cc:2229] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:06.837399 19133 pending_rounds.cc:62] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: Trying to abort 1 pending ops.
I20240502 01:59:06.837469 19133 pending_rounds.cc:68] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: Aborting op as it isn't in flight: id { term: 1 index: 7 } timestamp: 7023063640616800256 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "e2f4ca025890492880608c7975650969" old_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } } new_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } } }
I20240502 01:59:06.837635 19133 raft_consensus.cc:2894] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Aborting config change with OpId 1.7: Aborted: Op aborted
I20240502 01:59:06.837723 19133 raft_consensus.cc:2866] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 01:59:06.837785 19133 raft_consensus.cc:2258] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:06.838872 18624 raft_consensus.cc:1050] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
I20240502 01:59:06.839291 18624 consensus_queue.cc:237] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:06.840955 15466 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.841356 18151 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.841993 17105 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.843091 18907 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.844617 15331 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.844843 16837 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.845139 18907 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.845436 18151 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.850149 18782 ts_tablet_manager.cc:902] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:06.850536 18782 tablet_copy_client.cc:308] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:34323
I20240502 01:59:06.851119 15488 tablet_copy_service.cc:140] P 7b66a8940fe54e21bec1b2db25b2fed5: Received BeginTabletCopySession request for tablet 3f938c5af51a43c9a23b753c4991e41d from peer 3b8d615c71e44ff887e0f037c3efcff7 ({username='slave'} at 127.7.221.72:51521)
I20240502 01:59:06.851215 15488 tablet_copy_service.cc:161] P 7b66a8940fe54e21bec1b2db25b2fed5: Beginning new tablet copy session on tablet 3f938c5af51a43c9a23b753c4991e41d from peer 3b8d615c71e44ff887e0f037c3efcff7 at {username='slave'} at 127.7.221.72:51521: session id = 3b8d615c71e44ff887e0f037c3efcff7-3f938c5af51a43c9a23b753c4991e41d
W20240502 01:59:06.851645 19027 consensus_queue.cc:1159] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad [NON_LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "d4e5422f66454511ae5a50ec6ab134ad" status { last_received { term: 1 index: 6 } last_committed_idx: 5 last_received_current_leader { term: 1 index: 6 } }
I20240502 01:59:06.852051 15488 tablet_copy_source_session.cc:205] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: Tablet Copy: opened 0 blocks and 1 log segments
W20240502 01:59:06.852488 19114 consensus_queue.cc:1159] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" status { last_received { term: 1 index: 7 } last_committed_idx: 6 last_received_current_leader { term: 1 index: 7 } }
I20240502 01:59:06.853091 18857 ts_tablet_manager.cc:1875] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.853116 19133 ts_tablet_manager.cc:1875] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.853669 19158 tablet_bootstrap.cc:492] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.853933 19158 tablet_bootstrap.cc:492] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap complete.
I20240502 01:59:06.854094 19158 ts_tablet_manager.cc:1367] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad: Time spent bootstrapping tablet: real 0.033s user 0.020s sys 0.003s
I20240502 01:59:06.854521 19158 raft_consensus.cc:348] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:06.854641 19158 raft_consensus.cc:729] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d4e5422f66454511ae5a50ec6ab134ad, State: Initialized, Role: LEARNER
I20240502 01:59:06.854756 19158 consensus_queue.cc:260] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:06.855221 18782 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3f938c5af51a43c9a23b753c4991e41d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.855226 19133 ts_tablet_manager.cc:1888] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:06.855285 19133 log.cc:1196] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/e2f4ca025890492880608c7975650969
I20240502 01:59:06.856442 18857 ts_tablet_manager.cc:1888] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 01:59:06.856527 18857 log.cc:1196] T fc07e430e3e74e3885f263881bd04976 P d4e5422f66454511ae5a50ec6ab134ad: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal/wals/fc07e430e3e74e3885f263881bd04976
I20240502 01:59:06.857270 14901 catalog_manager.cc:4830] TS d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): tablet fc07e430e3e74e3885f263881bd04976 (table rebalance_test_table_1 [id=1c35c23ce11f4c249032c42d4c499e42]) successfully deleted
I20240502 01:59:06.857379 18857 tablet_replica.cc:330] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: stopping tablet replica
W20240502 01:59:06.857343 19102 consensus_queue.cc:1159] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" status { last_received { term: 1 index: 6 } last_committed_idx: 5 last_received_current_leader { term: 1 index: 6 } }
I20240502 01:59:06.857527 18782 tablet_copy_client.cc:783] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.857681 18782 tablet_copy_client.cc:647] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.858104 18857 raft_consensus.cc:2229] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Raft consensus shutting down.
I20240502 01:59:06.858196 18857 pending_rounds.cc:62] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: Trying to abort 1 pending ops.
I20240502 01:59:06.858248 17244 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.858292 18999 ts_tablet_manager.cc:1875] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.858233 18857 pending_rounds.cc:68] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: Aborting op as it isn't in flight: id { term: 1 index: 7 } timestamp: 7023063640616800256 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "e2f4ca025890492880608c7975650969" old_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } } new_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } } }
I20240502 01:59:06.858354 18857 raft_consensus.cc:2894] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Aborting config change with OpId 1.7: Aborted: Op aborted
I20240502 01:59:06.858413 18857 raft_consensus.cc:2866] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 01:59:06.858458 18857 raft_consensus.cc:2258] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Raft consensus is shut down!
I20240502 01:59:06.858196 19158 ts_tablet_manager.cc:1398] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad: Time spent starting tablet: real 0.004s user 0.004s sys 0.000s
I20240502 01:59:06.859105 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session d4e5422f66454511ae5a50ec6ab134ad-cd00409d47954ee2b7555e7ff09a6e28 received from {username='slave'} at 127.7.221.70:59193
I20240502 01:59:06.859176 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session d4e5422f66454511ae5a50ec6ab134ad-cd00409d47954ee2b7555e7ff09a6e28 on tablet cd00409d47954ee2b7555e7ff09a6e28 with peer d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:06.860383 18999 ts_tablet_manager.cc:1888] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 01:59:06.860455 18999 log.cc:1196] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/wals/d24986c2ebab46f086e3f1f5a53af26b
I20240502 01:59:06.861231 17243 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.861279 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:06.861282 14901 catalog_manager.cc:4830] TS 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): tablet d24986c2ebab46f086e3f1f5a53af26b (table rebalance_test_table_8 [id=bf7d5082701149bea6b9e5c480f53c10]) successfully deleted
I20240502 01:59:06.862080 19074 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.863058 17243 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.864110 18857 ts_tablet_manager.cc:1875] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.866420 18857 ts_tablet_manager.cc:1888] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:06.866568 18857 log.cc:1196] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal/wals/e2f4ca025890492880608c7975650969
I20240502 01:59:06.867179 14901 catalog_manager.cc:4830] TS d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): tablet e2f4ca025890492880608c7975650969 (table rebalance_test_table_0 [id=9736e7c5365849759564e386d208932f]) successfully deleted
I20240502 01:59:06.867703 18839 raft_consensus.cc:1050] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 to VOTER
I20240502 01:59:06.868120 18839 consensus_queue.cc:237] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:06.868147 15329 consensus_queue.cc:237] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:06.868292 17243 raft_consensus.cc:858] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }.
Pending config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }:
New config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
W20240502 01:59:06.868350 17243 raft_consensus.cc:1565] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Could not prepare op 'term: 1 index: 8' and following 0 ops. Status for this op: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }.
Pending config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }:
New config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.868634 17243 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.869791 15061 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.871482 14900 catalog_manager.cc:4830] TS 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905): tablet e2f4ca025890492880608c7975650969 (table rebalance_test_table_0 [id=9736e7c5365849759564e386d208932f]) successfully deleted
I20240502 01:59:06.874369 15462 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.874687 15463 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.874856 18151 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.875275 17244 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.875464 15194 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.876016 18416 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.876174 18189 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.876827 18367 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.877094 18367 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.877463 15463 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.877621 18416 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.880034 15331 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.882082 18880 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.882730 19158 ts_tablet_manager.cc:902] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:06.883126 19158 tablet_copy_client.cc:308] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:34323
I20240502 01:59:06.884557 14914 catalog_manager.cc:5484] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 5 to 6, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.885537 19075 raft_consensus.cc:1050] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 to VOTER
I20240502 01:59:06.885897 19075 consensus_queue.cc:237] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.887079 17242 raft_consensus.cc:1203] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:06.887421 17106 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.888213 15194 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.888571 19034 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.888610 14909 catalog_manager.cc:5484] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 6 to 7, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:06.890337 17242 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.890861 19034 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.891144 16972 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
W20240502 01:59:06.891324 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ac5ec2c673cb4b988d1099e0442ec63f. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:06.891160 16974 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.891561 15488 tablet_copy_service.cc:140] P 7b66a8940fe54e21bec1b2db25b2fed5: Received BeginTabletCopySession request for tablet 3f938c5af51a43c9a23b753c4991e41d from peer d4e5422f66454511ae5a50ec6ab134ad ({username='slave'} at 127.7.221.70:44607)
I20240502 01:59:06.891659 15488 tablet_copy_service.cc:161] P 7b66a8940fe54e21bec1b2db25b2fed5: Beginning new tablet copy session on tablet 3f938c5af51a43c9a23b753c4991e41d from peer d4e5422f66454511ae5a50ec6ab134ad at {username='slave'} at 127.7.221.70:44607: session id = d4e5422f66454511ae5a50ec6ab134ad-3f938c5af51a43c9a23b753c4991e41d
I20240502 01:59:06.891916 17242 raft_consensus.cc:858] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }.
Pending config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }:
New config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
W20240502 01:59:06.891974 17242 raft_consensus.cc:1565] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Could not prepare op 'term: 1 index: 9' and following 0 ops. Status for this op: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }.
Pending config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }:
New config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.892149 17242 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.892511 18416 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.892628 15488 tablet_copy_source_session.cc:205] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.893026 17471 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.894194 16972 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.894812 18386 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.895594 19074 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.895903 17242 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.897915 15329 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.898221 17106 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.899118 19158 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3f938c5af51a43c9a23b753c4991e41d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.899646 17243 raft_consensus.cc:1203] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 01:59:06.900408 17224 tablet_service.cc:1514] Processing DeleteTablet for tablet 30620f4774b8414eb4e27bd5ecb6ecee with delete_type TABLET_DATA_TOMBSTONED (TS 3b8d615c71e44ff887e0f037c3efcff7 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:37116
I20240502 01:59:06.901731 19158 tablet_copy_client.cc:783] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.901885 19158 tablet_copy_client.cc:647] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.902139 14915 catalog_manager.cc:5484] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 4 to 5, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.903120 15463 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.903594 14906 catalog_manager.cc:5484] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 reported cstate change: config changed from index 8 to 9, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.905797 15194 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.907066 19158 tablet_copy_client.cc:523] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.908639 18818 ts_tablet_manager.cc:902] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:06.908859 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 30620f4774b8414eb4e27bd5ecb6ecee with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.909107 18818 tablet_copy_client.cc:308] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:06.909668 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet cd00409d47954ee2b7555e7ff09a6e28 from peer fc067ee690bb457e8109ad8e4eb43ab8 ({username='slave'} at 127.7.221.71:39719)
I20240502 01:59:06.909770 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet cd00409d47954ee2b7555e7ff09a6e28 from peer fc067ee690bb457e8109ad8e4eb43ab8 at {username='slave'} at 127.7.221.71:39719: session id = fc067ee690bb457e8109ad8e4eb43ab8-cd00409d47954ee2b7555e7ff09a6e28
I20240502 01:59:06.910698 15216 tablet_copy_source_session.cc:205] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.911456 18818 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd00409d47954ee2b7555e7ff09a6e28. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.911603 19158 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap starting.
I20240502 01:59:06.913692 18818 tablet_copy_client.cc:783] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.913826 18818 tablet_copy_client.cc:647] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.914278 18367 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.916865 17243 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.917930 17106 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.918025 15464 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.919586 18189 raft_consensus.cc:1050] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 to VOTER
I20240502 01:59:06.920002 18189 consensus_queue.cc:237] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:06.920253 15331 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.921146 17910 raft_consensus.cc:1013] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 3b8d615c71e44ff887e0f037c3efcff7: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:06.922391 17243 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.922521 15329 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.922902 17471 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.923457 15464 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.923949 17471 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.924113 17910 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.924994 17910 raft_consensus.cc:1013] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 3b8d615c71e44ff887e0f037c3efcff7: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:06.925587 18151 raft_consensus.cc:1050] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09: attempting to promote NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 to VOTER
I20240502 01:59:06.926062 18151 consensus_queue.cc:237] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:06.926956 15194 consensus_queue.cc:237] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:06.928045 17108 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.928645 18150 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.929199 16972 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.929694 17471 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.929677 14913 catalog_manager.cc:5484] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER, now has a pending config: VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68), VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66), VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67), VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69), NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72), VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71). New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } } pending_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.930894 15331 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.932309 18150 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.932533 15194 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.933130 18150 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20240502 01:59:06.933380 19121 consensus_queue.cc:1159] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "3b8d615c71e44ff887e0f037c3efcff7" status { last_received { term: 1 index: 7 } last_committed_idx: 6 last_received_current_leader { term: 1 index: 7 } }
I20240502 01:59:06.933503 17243 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.933892 18150 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.934895 19164 ts_tablet_manager.cc:1875] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.935406 15463 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.935799 17106 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.936261 17471 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.936334 15330 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.936533 17471 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.936745 17242 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.937165 17471 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.937206 19164 ts_tablet_manager.cc:1888] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:06.937265 19164 log.cc:1196] T e2f4ca025890492880608c7975650969 P 3b8d615c71e44ff887e0f037c3efcff7: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal/wals/e2f4ca025890492880608c7975650969
I20240502 01:59:06.938395 19072 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.938292 17471 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.938761 17106 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.939792 18150 raft_consensus.cc:1050] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09: attempting to promote NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 to VOTER
I20240502 01:59:06.940161 18150 consensus_queue.cc:237] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:06.940694 17243 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.942112 17105 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.942230 15463 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.942549 18624 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.942819 18150 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.942979 15329 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.943040 17106 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.940701 15330 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.944051 18367 raft_consensus.cc:1050] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 to VOTER
I20240502 01:59:06.944548 18150 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.944561 18367 consensus_queue.cc:237] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:06.947067 15329 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:06.947983 15331 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.948294 17910 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.949258 16835 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.950839 17910 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.953893 17243 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.953881 16972 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.954303 18189 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.954557 14915 catalog_manager.cc:5484] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 7 to 8, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER, no longer has a pending config: VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68), VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66), VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67), VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69), NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72), VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71). New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.955163 15466 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.956487 16840 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:06.957015 16838 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.957288 17104 raft_consensus.cc:1203] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:06.957844 17471 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:06.958704 18782 tablet_copy_client.cc:523] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:06.958909 16838 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:06.960031 18782 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap starting.
I20240502 01:59:06.960294 16838 raft_consensus.cc:858] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }.
Pending config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }:
New config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
W20240502 01:59:06.960358 16838 raft_consensus.cc:1565] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Could not prepare op 'term: 1 index: 9' and following 0 ops. Status for this op: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }.
Pending config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }:
New config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:06.960590 16838 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.963490 19075 raft_consensus.cc:1050] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER d4e5422f66454511ae5a50ec6ab134ad to VOTER
I20240502 01:59:06.963768 19158 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:06.963946 17106 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.964035 19158 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap complete.
I20240502 01:59:06.963968 19075 consensus_queue.cc:237] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:06.964339 19214 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:06.964591 18907 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.965700 15060 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.966100 18279 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.966670 15197 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.967314 18279 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.967180 17104 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.968302 18279 raft_consensus.cc:1050] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 to VOTER
I20240502 01:59:06.968624 18279 consensus_queue.cc:237] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:06.969252 15466 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.969494 17243 raft_consensus.cc:1203] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 01:59:06.970292 15057 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.970347 17104 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.970714 19073 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.971158 19034 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.971524 16840 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.971909 19075 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.970006 17242 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.972779 15465 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:06.973368 18839 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.966055 16972 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:06.964207 19158 ts_tablet_manager.cc:1367] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad: Time spent bootstrapping tablet: real 0.053s user 0.028s sys 0.000s
I20240502 01:59:06.974289 19075 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:06.975520 15329 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.975911 17105 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.974649 19158 raft_consensus.cc:348] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.976342 19158 raft_consensus.cc:729] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d4e5422f66454511ae5a50ec6ab134ad, State: Initialized, Role: LEARNER
I20240502 01:59:06.976442 19158 consensus_queue.cc:260] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 1.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:06.977268 19158 ts_tablet_manager.cc:1398] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 01:59:06.977684 15488 tablet_copy_service.cc:342] P 7b66a8940fe54e21bec1b2db25b2fed5: Request end of tablet copy session d4e5422f66454511ae5a50ec6ab134ad-3f938c5af51a43c9a23b753c4991e41d received from {username='slave'} at 127.7.221.70:44607
I20240502 01:59:06.977772 15488 tablet_copy_service.cc:433] P 7b66a8940fe54e21bec1b2db25b2fed5: ending tablet copy session d4e5422f66454511ae5a50ec6ab134ad-3f938c5af51a43c9a23b753c4991e41d on tablet 3f938c5af51a43c9a23b753c4991e41d with peer d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:06.978137 14915 catalog_manager.cc:5484] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad reported cstate change: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:06.978440 14913 catalog_manager.cc:5484] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 8 to 9, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.979534 16836 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:06.979003 17242 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.980288 18150 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.981027 16836 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.981236 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a901eb3063ea4911a822552db01d365d with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:06.982582 17910 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:06.982497 16836 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.983325 15465 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.984593 19220 ts_tablet_manager.cc:902] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:06.984643 15465 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.985113 19220 tablet_copy_client.cc:308] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:34323
I20240502 01:59:06.986151 15488 tablet_copy_service.cc:140] P 7b66a8940fe54e21bec1b2db25b2fed5: Received BeginTabletCopySession request for tablet 3f938c5af51a43c9a23b753c4991e41d from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:53377)
I20240502 01:59:06.986248 15488 tablet_copy_service.cc:161] P 7b66a8940fe54e21bec1b2db25b2fed5: Beginning new tablet copy session on tablet 3f938c5af51a43c9a23b753c4991e41d from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:53377: session id = 84987ad9e4a54d96af8569e4aaf0ed64-3f938c5af51a43c9a23b753c4991e41d
I20240502 01:59:06.987118 15488 tablet_copy_source_session.cc:205] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:06.988649 16838 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:06.990943 15329 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:06.991343 19075 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:06.993324 19220 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3f938c5af51a43c9a23b753c4991e41d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:06.994539 15309 tablet_service.cc:1514] Processing DeleteTablet for tablet a901eb3063ea4911a822552db01d365d with delete_type TABLET_DATA_TOMBSTONED (TS f77c2a969bde45b49ac1bbbe242ff171 not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:43766
I20240502 01:59:06.994796 18851 tablet_replica.cc:330] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171: stopping tablet replica
I20240502 01:59:06.994935 18851 raft_consensus.cc:2229] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:06.995110 18851 raft_consensus.cc:2258] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:06.995819 19220 tablet_copy_client.cc:783] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:06.995965 19220 tablet_copy_client.cc:647] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:06.995884 14913 catalog_manager.cc:5484] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 reported cstate change: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.996564 18851 ts_tablet_manager.cc:1875] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:06.997228 15328 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:06.998430 18386 raft_consensus.cc:1013] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: attempt to promote peer fc067ee690bb457e8109ad8e4eb43ab8: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:06.998811 18851 ts_tablet_manager.cc:1888] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:06.998948 18851 log.cc:1196] T a901eb3063ea4911a822552db01d365d P f77c2a969bde45b49ac1bbbe242ff171: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/a901eb3063ea4911a822552db01d365d
I20240502 01:59:07.002247 14902 catalog_manager.cc:4830] TS 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): tablet e2f4ca025890492880608c7975650969 (table rebalance_test_table_0 [id=9736e7c5365849759564e386d208932f]) successfully deleted
W20240502 01:59:07.002388 14902 catalog_manager.cc:4810] TS 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): delete failed for tablet 30620f4774b8414eb4e27bd5ecb6ecee because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 30620f4774b8414eb4e27bd5ecb6ecee already in progress: copying tablet
I20240502 01:59:07.008730 18367 raft_consensus.cc:1013] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer d4e5422f66454511ae5a50ec6ab134ad: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:07.009687 15465 consensus_queue.cc:237] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.010841 14900 catalog_manager.cc:4830] TS f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): tablet a901eb3063ea4911a822552db01d365d (table rebalance_test_table_9 [id=12ee6dc62f814bd7b3c13bee4495d9e0]) successfully deleted
I20240502 01:59:07.011214 17244 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:07.011662 19073 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.011826 15197 consensus_queue.cc:237] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:07.012002 17108 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:07.012849 15329 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:07.013331 19034 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:07.013917 15464 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:07.013932 17108 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:07.014045 16838 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:07.014407 19034 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:07.014530 19214 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:07.014899 19214 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:07.016009 17244 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:07.016598 19073 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.016677 15329 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.016815 15197 consensus_queue.cc:237] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:07.017541 17910 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:07.018193 17910 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:07.018352 16974 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.018815 17910 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.018966 17108 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.019668 17244 raft_consensus.cc:1261] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.020009 18367 consensus_queue.cc:1035] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.020445 17910 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.022492 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ac5ec2c673cb4b988d1099e0442ec63f with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.022744 17244 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.022871 16974 raft_consensus.cc:2932] T a901eb3063ea4911a822552db01d365d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.023159 14909 catalog_manager.cc:5484] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 9 to 10, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.025848 19214 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.025889 17242 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.027798 17243 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.028240 15057 consensus_queue.cc:237] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:07.028775 15464 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.030162 17104 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.030596 18150 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.032109 16820 tablet_service.cc:1514] Processing DeleteTablet for tablet ac5ec2c673cb4b988d1099e0442ec63f with delete_type TABLET_DATA_TOMBSTONED (TS 84987ad9e4a54d96af8569e4aaf0ed64 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:55224
I20240502 01:59:07.033392 19219 ts_tablet_manager.cc:902] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:07.033269 19034 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.033869 19214 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.034440 19034 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.035112 15462 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.035185 14913 catalog_manager.cc:5484] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7 reported cstate change: config changed from index 9 to 10, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.035523 14913 catalog_manager.cc:5484] T a901eb3063ea4911a822552db01d365d P 3b8d615c71e44ff887e0f037c3efcff7 reported cstate change: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.036947 15060 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.036962 15329 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.038451 19219 tablet_copy_client.cc:308] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:07.038869 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet f97c5c9bb4c84957857efc76d1c7717f from peer 3b8d615c71e44ff887e0f037c3efcff7 ({username='slave'} at 127.7.221.72:57673)
I20240502 01:59:07.038954 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet f97c5c9bb4c84957857efc76d1c7717f from peer 3b8d615c71e44ff887e0f037c3efcff7 at {username='slave'} at 127.7.221.72:57673: session id = 3b8d615c71e44ff887e0f037c3efcff7-f97c5c9bb4c84957857efc76d1c7717f
I20240502 01:59:07.039003 16974 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.039855 15216 tablet_copy_source_session.cc:205] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:07.039912 15057 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.040594 16974 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.040751 19219 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f97c5c9bb4c84957857efc76d1c7717f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:07.040977 17243 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.042135 19214 raft_consensus.cc:1050] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER d4e5422f66454511ae5a50ec6ab134ad to VOTER
I20240502 01:59:07.042505 19214 consensus_queue.cc:237] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:07.042917 15446 tablet_service.cc:1514] Processing DeleteTablet for tablet a901eb3063ea4911a822552db01d365d with delete_type TABLET_DATA_TOMBSTONED (TS 7b66a8940fe54e21bec1b2db25b2fed5 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:60890
I20240502 01:59:07.042969 17108 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.043228 18999 tablet_replica.cc:330] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: stopping tablet replica
I20240502 01:59:07.043355 18999 raft_consensus.cc:2229] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.043524 18999 raft_consensus.cc:2258] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.044009 16974 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.044118 18999 ts_tablet_manager.cc:1875] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.044072 18386 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.044464 17471 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.046001 18999 ts_tablet_manager.cc:1888] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 01:59:07.046067 18999 log.cc:1196] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/wals/a901eb3063ea4911a822552db01d365d
I20240502 01:59:07.046576 15329 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.046801 17240 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.047243 17243 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.047580 18367 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.048471 17105 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.049101 14915 catalog_manager.cc:5484] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 reported cstate change: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.051172 15464 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.051223 14913 catalog_manager.cc:5484] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.051367 15462 consensus_queue.cc:237] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.052676 17240 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:07.052975 17105 raft_consensus.cc:1261] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:07.053004 18296 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:07.053174 15197 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.053383 18296 consensus_queue.cc:1035] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:07.054646 18189 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.054664 18296 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.055603 14901 catalog_manager.cc:4830] TS 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): tablet a901eb3063ea4911a822552db01d365d (table rebalance_test_table_9 [id=12ee6dc62f814bd7b3c13bee4495d9e0]) successfully deleted
I20240502 01:59:07.055974 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ac5ec2c673cb4b988d1099e0442ec63f with cas_config_opid_index 10: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.056255 17240 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.056504 14909 catalog_manager.cc:5484] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.056787 18367 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.058463 15462 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.059600 15057 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.059700 17105 raft_consensus.cc:2932] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.060027 17471 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.060075 16836 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.060690 15057 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.060935 19219 tablet_copy_client.cc:783] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:07.061123 19219 tablet_copy_client.cc:647] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:07.060465 18386 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.061549 17240 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.062186 18151 raft_consensus.cc:1050] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09: attempting to promote NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 to VOTER
I20240502 01:59:07.062090 15057 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.062615 18151 consensus_queue.cc:237] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.064509 15329 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.064932 18150 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.066254 16838 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.066502 19219 tablet_copy_client.cc:523] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:07.066941 15465 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.067147 15197 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.067327 16974 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.067744 18150 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.067927 19219 tablet_bootstrap.cc:492] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap starting.
I20240502 01:59:07.067988 18150 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.069097 17243 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.069412 17106 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.069566 18150 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.069969 14909 catalog_manager.cc:5484] T ac5ec2c673cb4b988d1099e0442ec63f P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 10 to 11, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.071344 18624 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.073494 17104 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
W20240502 01:59:07.076575 19215 log.cc:924] Time spent T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09: Append to log took a long time: real 0.078s user 0.000s sys 0.000s
I20240502 01:59:07.076861 15309 tablet_service.cc:1514] Processing DeleteTablet for tablet 856f623374824d94913db0cb9faa25d1 with delete_type TABLET_DATA_TOMBSTONED (TS f77c2a969bde45b49ac1bbbe242ff171 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:43766
I20240502 01:59:07.077070 18851 tablet_replica.cc:330] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171: stopping tablet replica
I20240502 01:59:07.077194 18851 raft_consensus.cc:2229] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.077414 18851 raft_consensus.cc:2258] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.078043 18851 ts_tablet_manager.cc:1875] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.080075 18851 ts_tablet_manager.cc:1888] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 01:59:07.080235 18851 log.cc:1196] T 856f623374824d94913db0cb9faa25d1 P f77c2a969bde45b49ac1bbbe242ff171: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/856f623374824d94913db0cb9faa25d1
I20240502 01:59:07.080914 16840 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.081455 19071 raft_consensus.cc:1050] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
I20240502 01:59:07.081835 19071 consensus_queue.cc:237] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:07.083384 15329 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.083798 19034 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.084100 16840 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
W20240502 01:59:07.087069 14902 catalog_manager.cc:4797] TS 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): delete failed for tablet ac5ec2c673cb4b988d1099e0442ec63f because the tablet was not found. No further retry: Not found: Tablet not found: ac5ec2c673cb4b988d1099e0442ec63f
I20240502 01:59:07.087944 15057 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.088455 19073 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.089277 16836 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.089648 19073 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.090869 17103 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.091403 16974 raft_consensus.cc:1203] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:07.092137 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e00d1ed5cde044cb9bdebf787590261a with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.093377 15465 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.095419 18624 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.095682 17105 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.096035 18150 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.097780 15329 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.098697 19034 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.099040 15197 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.102067 14909 catalog_manager.cc:5484] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.103680 14912 catalog_manager.cc:5484] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.103724 17105 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.105141 17243 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.106985 19034 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.108495 17104 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.109017 15329 consensus_queue.cc:237] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:07.110428 16836 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
W20240502 01:59:07.111074 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 490e37fac0844a08a07a4b281bc33bf9. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:07.112232 15465 raft_consensus.cc:1261] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:07.112658 18863 consensus_queue.cc:1035] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:07.112751 15197 raft_consensus.cc:1261] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:07.113124 18863 consensus_queue.cc:1035] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:07.112668 15329 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.113344 14912 catalog_manager.cc:5484] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.116554 15057 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.122066 18818 tablet_copy_client.cc:523] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:07.122126 15194 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.123431 18818 tablet_bootstrap.cc:492] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap starting.
I20240502 01:59:07.125751 15309 tablet_service.cc:1514] Processing DeleteTablet for tablet ac5ec2c673cb4b988d1099e0442ec63f with delete_type TABLET_DATA_TOMBSTONED (TS f77c2a969bde45b49ac1bbbe242ff171 not found in new config with opid_index 11) from {username='slave'} at 127.0.0.1:43766
I20240502 01:59:07.126164 18851 tablet_replica.cc:330] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171: stopping tablet replica
I20240502 01:59:07.126320 18851 raft_consensus.cc:2229] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.126536 18851 raft_consensus.cc:2258] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.127274 18851 ts_tablet_manager.cc:1875] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.128078 18637 tablet_bootstrap.cc:492] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:07.128336 18637 tablet_bootstrap.cc:492] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:07.128512 18637 ts_tablet_manager.cc:1367] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.536s user 0.026s sys 0.000s
I20240502 01:59:07.128984 18637 raft_consensus.cc:348] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:07.129099 18637 raft_consensus.cc:729] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:07.129199 18637 consensus_queue.cc:260] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:07.129485 18637 ts_tablet_manager.cc:1398] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:07.129485 18851 ts_tablet_manager.cc:1888] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.10
I20240502 01:59:07.129542 18851 log.cc:1196] T ac5ec2c673cb4b988d1099e0442ec63f P f77c2a969bde45b49ac1bbbe242ff171: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/ac5ec2c673cb4b988d1099e0442ec63f
I20240502 01:59:07.130026 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-091d27d51acc4db0a1c7d20deeff5c10 received from {username='slave'} at 127.7.221.69:51343
I20240502 01:59:07.130106 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-091d27d51acc4db0a1c7d20deeff5c10 on tablet 091d27d51acc4db0a1c7d20deeff5c10 with peer 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:07.136442 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a901eb3063ea4911a822552db01d365d with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.136682 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 856f623374824d94913db0cb9faa25d1 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.137130 15309 tablet_service.cc:1514] Processing DeleteTablet for tablet e00d1ed5cde044cb9bdebf787590261a with delete_type TABLET_DATA_TOMBSTONED (TS f77c2a969bde45b49ac1bbbe242ff171 not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:43766
I20240502 01:59:07.137408 18851 tablet_replica.cc:330] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171: stopping tablet replica
I20240502 01:59:07.137553 18851 raft_consensus.cc:2229] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.137753 18851 raft_consensus.cc:2258] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.138448 18851 ts_tablet_manager.cc:1875] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.121621 14915 catalog_manager.cc:5484] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 4 to 5, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:07.141163 18851 ts_tablet_manager.cc:1888] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:07.141297 18851 log.cc:1196] T e00d1ed5cde044cb9bdebf787590261a P f77c2a969bde45b49ac1bbbe242ff171: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/e00d1ed5cde044cb9bdebf787590261a
I20240502 01:59:07.141513 18417 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.142035 15195 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.142692 18737 tablet_bootstrap.cc:492] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:07.143038 18737 tablet_bootstrap.cc:492] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap complete.
I20240502 01:59:07.143213 18737 ts_tablet_manager.cc:1367] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8: Time spent bootstrapping tablet: real 0.524s user 0.023s sys 0.003s
I20240502 01:59:07.143692 18737 raft_consensus.cc:348] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:07.147753 14915 catalog_manager.cc:5031] ChangeConfig:REMOVE_PEER RPC for tablet 6d5cffb7fcc54118861463e3b1fa4a8d with cas_config_opid_index 6: aborting the task: latest config opid_index 7; task opid_index 6
I20240502 01:59:07.148236 14914 catalog_manager.cc:5484] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 reported cstate change: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.151101 14908 catalog_manager.cc:5484] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:07.152377 18712 tablet_bootstrap.cc:492] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:07.152738 18712 tablet_bootstrap.cc:492] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap complete.
I20240502 01:59:07.152982 18712 ts_tablet_manager.cc:1367] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7: Time spent bootstrapping tablet: real 0.376s user 0.028s sys 0.000s
I20240502 01:59:07.154397 14900 catalog_manager.cc:4830] TS f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): tablet 856f623374824d94913db0cb9faa25d1 (table rebalance_test_table_9 [id=12ee6dc62f814bd7b3c13bee4495d9e0]) successfully deleted
I20240502 01:59:07.154533 14900 catalog_manager.cc:4830] TS f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): tablet ac5ec2c673cb4b988d1099e0442ec63f (table rebalance_test_table_5 [id=3bc69a0a1e474b9fac7330d68770bfbe]) successfully deleted
I20240502 01:59:07.154652 14900 catalog_manager.cc:4830] TS f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): tablet e00d1ed5cde044cb9bdebf787590261a (table rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]) successfully deleted
I20240502 01:59:07.160435 15465 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.168116 15195 consensus_queue.cc:237] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:07.169808 15057 consensus_queue.cc:237] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:07.170308 16836 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:07.170825 15195 consensus_queue.cc:237] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:07.171063 19214 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:07.171811 16840 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.172222 18151 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.173684 16974 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.173844 15057 consensus_queue.cc:237] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.174005 15465 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.175084 18311 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.175251 18517 tablet_bootstrap.cc:492] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:07.175540 18517 tablet_bootstrap.cc:492] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:07.175532 15329 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.175750 18517 ts_tablet_manager.cc:1367] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.635s user 0.028s sys 0.000s
I20240502 01:59:07.176263 18517 raft_consensus.cc:348] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:07.176383 18517 raft_consensus.cc:729] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:07.176483 18517 consensus_queue.cc:260] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:07.176916 18517 ts_tablet_manager.cc:1398] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 01:59:07.171381 17104 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.177667 15465 consensus_queue.cc:237] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:07.177794 18319 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.172956 17105 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:07.176836 17106 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.178309 18737 raft_consensus.cc:729] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fc067ee690bb457e8109ad8e4eb43ab8, State: Initialized, Role: LEARNER
I20240502 01:59:07.178460 18737 consensus_queue.cc:260] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:07.178915 18737 ts_tablet_manager.cc:1398] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8: Time spent starting tablet: real 0.036s user 0.001s sys 0.000s
I20240502 01:59:07.179097 19214 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:07.179487 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-6c4e85e7050b4dc8a4ce6bba818854a2 received from {username='slave'} at 127.7.221.69:43177
I20240502 01:59:07.179571 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-6c4e85e7050b4dc8a4ce6bba818854a2 on tablet 6c4e85e7050b4dc8a4ce6bba818854a2 with peer 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:07.179661 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-ce85785e670a4264b44797bb27cb4ecb received from {username='slave'} at 127.7.221.71:41349
I20240502 01:59:07.179746 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-ce85785e670a4264b44797bb27cb4ecb on tablet ce85785e670a4264b44797bb27cb4ecb with peer fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:07.179690 18712 raft_consensus.cc:348] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:07.179834 18712 raft_consensus.cc:729] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b8d615c71e44ff887e0f037c3efcff7, State: Initialized, Role: LEARNER
I20240502 01:59:07.179956 18712 consensus_queue.cc:260] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 1.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:07.180747 18907 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.181056 18907 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.181874 17242 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.181874 15329 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.182137 17224 tablet_service.cc:1514] Processing DeleteTablet for tablet 30620f4774b8414eb4e27bd5ecb6ecee with delete_type TABLET_DATA_TOMBSTONED (Replica with old config index 6 (current committed config index is 7)) from {username='slave'} at 127.0.0.1:37116
I20240502 01:59:07.182271 18296 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240502 01:59:07.182538 14902 catalog_manager.cc:4810] TS 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): delete failed for tablet 30620f4774b8414eb4e27bd5ecb6ecee because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 30620f4774b8414eb4e27bd5ecb6ecee already in progress: copying tablet
I20240502 01:59:07.182727 18319 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.183094 19214 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.184180 16838 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.184594 18296 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.185410 17106 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.185416 16840 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.185635 17103 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.186719 17108 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.187098 18767 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.187330 18839 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.187461 17471 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.188024 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e00d1ed5cde044cb9bdebf787590261a with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.188000 18151 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.188313 19220 tablet_copy_client.cc:523] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:07.188829 18712 ts_tablet_manager.cc:1398] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7: Time spent starting tablet: real 0.010s user 0.001s sys 0.000s
I20240502 01:59:07.189185 18767 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.158149 18782 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:07.189957 18782 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap complete.
I20240502 01:59:07.190140 18782 ts_tablet_manager.cc:1367] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7: Time spent bootstrapping tablet: real 0.230s user 0.027s sys 0.001s
I20240502 01:59:07.190358 17242 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.190621 17224 tablet_service.cc:1514] Processing DeleteTablet for tablet 30620f4774b8414eb4e27bd5ecb6ecee with delete_type TABLET_DATA_TOMBSTONED (Replica with old config index 6 (current committed config index is 7)) from {username='slave'} at 127.0.0.1:37116
I20240502 01:59:07.190807 19164 tablet_replica.cc:330] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7: stopping tablet replica
I20240502 01:59:07.190930 19164 raft_consensus.cc:2229] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Raft consensus shutting down.
I20240502 01:59:07.191025 19164 raft_consensus.cc:2258] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Raft consensus is shut down!
I20240502 01:59:07.170660 17243 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:07.173105 17240 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.191733 18839 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:07.192019 19164 ts_tablet_manager.cc:1875] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.192070 18839 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.193524 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-30620f4774b8414eb4e27bd5ecb6ecee received from {username='slave'} at 127.7.221.72:47237
I20240502 01:59:07.193598 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-30620f4774b8414eb4e27bd5ecb6ecee on tablet 30620f4774b8414eb4e27bd5ecb6ecee with peer 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:07.194056 19164 ts_tablet_manager.cc:1888] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 01:59:07.194124 19164 log.cc:1196] T 30620f4774b8414eb4e27bd5ecb6ecee P 3b8d615c71e44ff887e0f037c3efcff7: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal/wals/30620f4774b8414eb4e27bd5ecb6ecee
I20240502 01:59:07.194373 15329 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.194766 17471 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.195371 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:07.196547 15328 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.197355 16974 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.198078 17243 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.200078 17242 raft_consensus.cc:2932] T 856f623374824d94913db0cb9faa25d1 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.200585 14902 catalog_manager.cc:4830] TS 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): tablet 30620f4774b8414eb4e27bd5ecb6ecee (table rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]) successfully deleted
I20240502 01:59:07.201115 17103 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.203462 15465 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.203871 14913 catalog_manager.cc:5484] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7 reported cstate change: config changed from index 7 to 8, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.205524 14906 catalog_manager.cc:5484] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 10 to 11, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.205834 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 856f623374824d94913db0cb9faa25d1 with cas_config_opid_index 10: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.205865 14906 catalog_manager.cc:5484] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.205984 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 81c8fe21ec7545e79450532b43a61ee7 with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.208796 17104 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.215526 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6d5cffb7fcc54118861463e3b1fa4a8d with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.215605 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:07.217762 15174 tablet_service.cc:1514] Processing DeleteTablet for tablet 1eb15147ff6c4d0b83b12dec736fc576 with delete_type TABLET_DATA_TOMBSTONED (Replica with old config index 7 (current committed config index is 8)) from {username='slave'} at 127.0.0.1:37206
I20240502 01:59:07.204219 19220 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
I20240502 01:59:07.205209 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:07.218494 19133 tablet_replica.cc:330] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: stopping tablet replica
I20240502 01:59:07.218680 19133 raft_consensus.cc:2229] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.218855 19133 raft_consensus.cc:2258] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.220161 19133 ts_tablet_manager.cc:1875] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.217320 14909 catalog_manager.cc:5484] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.222095 19133 ts_tablet_manager.cc:1888] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:07.222157 19133 log.cc:1196] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/1eb15147ff6c4d0b83b12dec736fc576
I20240502 01:59:07.223027 14900 catalog_manager.cc:4830] TS 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905): tablet 1eb15147ff6c4d0b83b12dec736fc576 (table rebalance_test_table_4 [id=0fd3b28283df457db390a2882ebcfef5]) successfully deleted
I20240502 01:59:07.205615 16836 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.232867 18311 raft_consensus.cc:1050] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
I20240502 01:59:07.207948 16838 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.233224 18311 consensus_queue.cc:237] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:07.236266 15195 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:07.236853 18319 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:07.238430 16836 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:07.238931 18907 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:07.240494 15328 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:07.240988 18907 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:07.244971 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1eb15147ff6c4d0b83b12dec736fc576 with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.245831 15040 tablet_service.cc:1514] Processing DeleteTablet for tablet 6d5cffb7fcc54118861463e3b1fa4a8d with delete_type TABLET_DATA_TOMBSTONED (TS ddae2a636423489693c12b7d7d166c09 not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:53384
I20240502 01:59:07.246019 19132 tablet_replica.cc:330] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09: stopping tablet replica
I20240502 01:59:07.246134 19132 raft_consensus.cc:2229] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.246354 19132 raft_consensus.cc:2258] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.246388 15446 tablet_service.cc:1514] Processing DeleteTablet for tablet 856f623374824d94913db0cb9faa25d1 with delete_type TABLET_DATA_TOMBSTONED (TS 7b66a8940fe54e21bec1b2db25b2fed5 not found in new config with opid_index 11) from {username='slave'} at 127.0.0.1:60890
I20240502 01:59:07.246455 15444 tablet_service.cc:1514] Processing DeleteTablet for tablet e00d1ed5cde044cb9bdebf787590261a with delete_type TABLET_DATA_TOMBSTONED (TS 7b66a8940fe54e21bec1b2db25b2fed5 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:60890
I20240502 01:59:07.246363 15057 consensus_queue.cc:237] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.246598 18999 tablet_replica.cc:330] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5: stopping tablet replica
I20240502 01:59:07.246742 18999 raft_consensus.cc:2229] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.246989 18999 raft_consensus.cc:2258] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.247149 19132 ts_tablet_manager.cc:1875] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.247665 18999 ts_tablet_manager.cc:1875] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.249610 19132 ts_tablet_manager.cc:1888] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:07.249684 19132 log.cc:1196] T 6d5cffb7fcc54118861463e3b1fa4a8d P ddae2a636423489693c12b7d7d166c09: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/6d5cffb7fcc54118861463e3b1fa4a8d
I20240502 01:59:07.249861 18999 ts_tablet_manager.cc:1888] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.10
I20240502 01:59:07.249922 18999 log.cc:1196] T 856f623374824d94913db0cb9faa25d1 P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/wals/856f623374824d94913db0cb9faa25d1
I20240502 01:59:07.250680 18999 tablet_replica.cc:330] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5: stopping tablet replica
I20240502 01:59:07.250742 18818 tablet_bootstrap.cc:492] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:07.250804 18999 raft_consensus.cc:2229] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.250865 17242 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.250994 18999 raft_consensus.cc:2258] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.251024 18818 tablet_bootstrap.cc:492] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap complete.
I20240502 01:59:07.251191 18818 ts_tablet_manager.cc:1367] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8: Time spent bootstrapping tablet: real 0.128s user 0.028s sys 0.000s
I20240502 01:59:07.251731 18386 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.251703 18818 raft_consensus.cc:348] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:07.251919 18818 raft_consensus.cc:729] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fc067ee690bb457e8109ad8e4eb43ab8, State: Initialized, Role: LEARNER
I20240502 01:59:07.251957 17104 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.252384 18999 ts_tablet_manager.cc:1875] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.252418 18386 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.253429 18907 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.254545 18999 ts_tablet_manager.cc:1888] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 01:59:07.254619 18999 log.cc:1196] T e00d1ed5cde044cb9bdebf787590261a P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/wals/e00d1ed5cde044cb9bdebf787590261a
I20240502 01:59:07.256021 18782 raft_consensus.cc:348] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:07.256177 18782 raft_consensus.cc:729] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b8d615c71e44ff887e0f037c3efcff7, State: Initialized, Role: LEARNER
I20240502 01:59:07.256321 18782 consensus_queue.cc:260] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 1.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:07.256762 18782 ts_tablet_manager.cc:1398] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7: Time spent starting tablet: real 0.067s user 0.001s sys 0.000s
I20240502 01:59:07.257205 15488 tablet_copy_service.cc:342] P 7b66a8940fe54e21bec1b2db25b2fed5: Request end of tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-3f938c5af51a43c9a23b753c4991e41d received from {username='slave'} at 127.7.221.72:51521
I20240502 01:59:07.257273 15488 tablet_copy_service.cc:433] P 7b66a8940fe54e21bec1b2db25b2fed5: ending tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-3f938c5af51a43c9a23b753c4991e41d on tablet 3f938c5af51a43c9a23b753c4991e41d with peer 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:07.258193 17242 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.259645 14910 catalog_manager.cc:5484] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 8 to 9, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:07.260735 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:07.261063 14907 catalog_manager.cc:5484] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:07.262624 14901 catalog_manager.cc:4830] TS ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123): tablet 6d5cffb7fcc54118861463e3b1fa4a8d (table rebalance_test_table_6 [id=2739aac0519d438587e8ec91530f5579]) successfully deleted
I20240502 01:59:07.262773 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1eb15147ff6c4d0b83b12dec736fc576 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.263463 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:07.265281 15309 tablet_service.cc:1514] Processing DeleteTablet for tablet 1eb15147ff6c4d0b83b12dec736fc576 with delete_type TABLET_DATA_TOMBSTONED (TS f77c2a969bde45b49ac1bbbe242ff171 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:43766
I20240502 01:59:07.265473 18851 tablet_replica.cc:330] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171: stopping tablet replica
I20240502 01:59:07.265597 18851 raft_consensus.cc:2229] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.265789 18851 raft_consensus.cc:2258] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.266541 18851 ts_tablet_manager.cc:1875] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.267150 15194 consensus_queue.cc:237] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:07.268595 18851 ts_tablet_manager.cc:1888] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 01:59:07.268620 14901 catalog_manager.cc:4830] TS 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): tablet 856f623374824d94913db0cb9faa25d1 (table rebalance_test_table_9 [id=12ee6dc62f814bd7b3c13bee4495d9e0]) successfully deleted
I20240502 01:59:07.268656 18851 log.cc:1196] T 1eb15147ff6c4d0b83b12dec736fc576 P f77c2a969bde45b49ac1bbbe242ff171: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/1eb15147ff6c4d0b83b12dec736fc576
I20240502 01:59:07.268743 14901 catalog_manager.cc:4830] TS 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): tablet e00d1ed5cde044cb9bdebf787590261a (table rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]) successfully deleted
I20240502 01:59:07.269531 16974 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.270500 18189 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.270943 17243 raft_consensus.cc:1203] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.5->[1.6-1.6] Dedup: 1.6->[]
I20240502 01:59:07.274044 18189 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.276232 18319 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.276914 16974 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.278131 15195 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.278842 15328 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.279053 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 81c8fe21ec7545e79450532b43a61ee7 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.280293 17242 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.280740 18367 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.282050 15040 tablet_service.cc:1514] Processing DeleteTablet for tablet 81c8fe21ec7545e79450532b43a61ee7 with delete_type TABLET_DATA_TOMBSTONED (TS ddae2a636423489693c12b7d7d166c09 not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:53384
I20240502 01:59:07.282474 19132 tablet_replica.cc:330] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09: stopping tablet replica
I20240502 01:59:07.282649 19132 raft_consensus.cc:2229] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.282842 19132 raft_consensus.cc:2258] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.283252 17242 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.283363 16840 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.283577 19132 ts_tablet_manager.cc:1875] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.285717 19132 ts_tablet_manager.cc:1888] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:07.285799 19132 log.cc:1196] T 81c8fe21ec7545e79450532b43a61ee7 P ddae2a636423489693c12b7d7d166c09: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/81c8fe21ec7545e79450532b43a61ee7
I20240502 01:59:07.287778 17106 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.290413 18818 consensus_queue.cc:260] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:07.291319 19219 tablet_bootstrap.cc:492] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:07.291636 14900 catalog_manager.cc:4830] TS f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): tablet 1eb15147ff6c4d0b83b12dec736fc576 (table rebalance_test_table_4 [id=0fd3b28283df457db390a2882ebcfef5]) successfully deleted
I20240502 01:59:07.291960 15328 consensus_queue.cc:237] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:07.293457 15465 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.293613 15195 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.294019 18416 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.294027 14906 catalog_manager.cc:5484] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 4 to 5, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:07.294386 18416 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.295704 16974 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.296082 18416 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
W20240502 01:59:07.296556 15266 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:07.297452 18416 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.298360 15465 consensus_queue.cc:237] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:07.298849 16974 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:07.299625 17104 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.300195 18296 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.298355 15462 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:07.300390 14909 catalog_manager.cc:5484] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:07.300879 15195 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:07.301995 18296 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.303092 16838 raft_consensus.cc:1261] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.303408 19075 consensus_queue.cc:1035] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.304533 17104 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.304771 14906 catalog_manager.cc:5484] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 reported cstate change: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.305295 16838 raft_consensus.cc:2932] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.312896 18818 ts_tablet_manager.cc:1398] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8: Time spent starting tablet: real 0.062s user 0.001s sys 0.000s
I20240502 01:59:07.313364 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-cd00409d47954ee2b7555e7ff09a6e28 received from {username='slave'} at 127.7.221.71:39719
I20240502 01:59:07.313432 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-cd00409d47954ee2b7555e7ff09a6e28 on tablet cd00409d47954ee2b7555e7ff09a6e28 with peer fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:07.314298 19219 tablet_bootstrap.cc:492] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap complete.
I20240502 01:59:07.314548 19219 ts_tablet_manager.cc:1367] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7: Time spent bootstrapping tablet: real 0.247s user 0.015s sys 0.011s
I20240502 01:59:07.315114 19219 raft_consensus.cc:348] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:07.315330 19219 raft_consensus.cc:729] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b8d615c71e44ff887e0f037c3efcff7, State: Initialized, Role: LEARNER
I20240502 01:59:07.315485 19219 consensus_queue.cc:260] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:07.315948 19219 ts_tablet_manager.cc:1398] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 01:59:07.316422 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-f97c5c9bb4c84957857efc76d1c7717f received from {username='slave'} at 127.7.221.72:57673
I20240502 01:59:07.316488 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-f97c5c9bb4c84957857efc76d1c7717f on tablet f97c5c9bb4c84957857efc76d1c7717f with peer 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:07.316515 15309 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:07.323292 14909 catalog_manager.cc:5484] T 6d5cffb7fcc54118861463e3b1fa4a8d P 84987ad9e4a54d96af8569e4aaf0ed64 reported cstate change: config changed from index 8 to 9, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.325646 14901 catalog_manager.cc:4830] TS ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123): tablet 81c8fe21ec7545e79450532b43a61ee7 (table rebalance_test_table_6 [id=2739aac0519d438587e8ec91530f5579]) successfully deleted
I20240502 01:59:07.325718 15309 tablet_service.cc:1514] Processing DeleteTablet for tablet 6d5cffb7fcc54118861463e3b1fa4a8d with delete_type TABLET_DATA_TOMBSTONED (TS f77c2a969bde45b49ac1bbbe242ff171 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:43766
I20240502 01:59:07.326089 18851 tablet_replica.cc:330] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171: stopping tablet replica
I20240502 01:59:07.326215 18851 raft_consensus.cc:2229] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.326292 15444 tablet_service.cc:1514] Processing DeleteTablet for tablet 81c8fe21ec7545e79450532b43a61ee7 with delete_type TABLET_DATA_TOMBSTONED (TS 7b66a8940fe54e21bec1b2db25b2fed5 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:60890
I20240502 01:59:07.326371 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6d5cffb7fcc54118861463e3b1fa4a8d with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.326419 18851 raft_consensus.cc:2258] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.328855 18863 raft_consensus.cc:1050] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171: attempting to promote NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 to VOTER
I20240502 01:59:07.329234 18863 consensus_queue.cc:237] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:07.331018 17104 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.331097 15462 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.331269 17242 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.331470 18417 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.331650 18416 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.332967 18416 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.333999 15444 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:07.335017 15057 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.335260 15061 consensus_queue.cc:237] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:07.336634 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:07.336864 15195 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.336977 18416 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.337291 18319 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.337764 18851 ts_tablet_manager.cc:1875] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.339392 18319 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.340065 18851 ts_tablet_manager.cc:1888] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 01:59:07.340127 18851 log.cc:1196] T 6d5cffb7fcc54118861463e3b1fa4a8d P f77c2a969bde45b49ac1bbbe242ff171: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/6d5cffb7fcc54118861463e3b1fa4a8d
I20240502 01:59:07.340942 16838 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.341411 15195 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.341713 18999 tablet_replica.cc:330] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5: stopping tablet replica
I20240502 01:59:07.341842 18999 raft_consensus.cc:2229] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.342094 14900 catalog_manager.cc:4830] TS f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): tablet 6d5cffb7fcc54118861463e3b1fa4a8d (table rebalance_test_table_6 [id=2739aac0519d438587e8ec91530f5579]) successfully deleted
I20240502 01:59:07.342296 18999 raft_consensus.cc:2258] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.345137 17106 raft_consensus.cc:1203] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:07.345590 14909 catalog_manager.cc:5484] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 5 to 6, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.346645 18999 ts_tablet_manager.cc:1875] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.348702 15309 tablet_service.cc:1514] Processing DeleteTablet for tablet 77ae087802144c45924c1b79f561afe8 with delete_type TABLET_DATA_TOMBSTONED (TS f77c2a969bde45b49ac1bbbe242ff171 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:43766
I20240502 01:59:07.349233 18999 ts_tablet_manager.cc:1888] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 01:59:07.349344 18999 log.cc:1196] T 81c8fe21ec7545e79450532b43a61ee7 P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/wals/81c8fe21ec7545e79450532b43a61ee7
I20240502 01:59:07.349848 18851 tablet_replica.cc:330] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171: stopping tablet replica
I20240502 01:59:07.350028 18851 raft_consensus.cc:2229] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.350229 18851 raft_consensus.cc:2258] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.352121 14901 catalog_manager.cc:4830] TS 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): tablet 81c8fe21ec7545e79450532b43a61ee7 (table rebalance_test_table_6 [id=2739aac0519d438587e8ec91530f5579]) successfully deleted
I20240502 01:59:07.352523 18818 ts_tablet_manager.cc:902] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:07.352784 18851 ts_tablet_manager.cc:1875] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.353005 18818 tablet_copy_client.cc:308] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 01:59:07.353459 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet 6db29f2516e1486782be0312c9bcac8b from peer fc067ee690bb457e8109ad8e4eb43ab8 ({username='slave'} at 127.7.221.71:41349)
I20240502 01:59:07.353549 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet 6db29f2516e1486782be0312c9bcac8b from peer fc067ee690bb457e8109ad8e4eb43ab8 at {username='slave'} at 127.7.221.71:41349: session id = fc067ee690bb457e8109ad8e4eb43ab8-6db29f2516e1486782be0312c9bcac8b
I20240502 01:59:07.354415 15351 tablet_copy_source_session.cc:205] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:07.354987 16974 raft_consensus.cc:1203] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Deduplicated request from leader. Original: 1.5->[1.6-1.6] Dedup: 1.6->[]
I20240502 01:59:07.356074 18818 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6db29f2516e1486782be0312c9bcac8b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:07.356294 18851 ts_tablet_manager.cc:1888] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 01:59:07.356353 18851 log.cc:1196] T 77ae087802144c45924c1b79f561afe8 P f77c2a969bde45b49ac1bbbe242ff171: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/77ae087802144c45924c1b79f561afe8
I20240502 01:59:07.356866 14900 catalog_manager.cc:4830] TS f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): tablet 77ae087802144c45924c1b79f561afe8 (table rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]) successfully deleted
I20240502 01:59:07.358264 18818 tablet_copy_client.cc:783] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:07.358410 18818 tablet_copy_client.cc:647] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:07.359691 18417 raft_consensus.cc:1013] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: attempt to promote peer fc067ee690bb457e8109ad8e4eb43ab8: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:07.360950 18386 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.363690 16840 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.364722 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:07.373839 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 77ae087802144c45924c1b79f561afe8 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.374428 18880 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:07.374787 15061 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:07.377362 14912 catalog_manager.cc:5484] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: HEALTHY } } }
I20240502 01:59:07.377748 17106 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:07.377939 17242 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:07.381062 18416 raft_consensus.cc:1050] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171: attempting to promote NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 to VOTER
I20240502 01:59:07.381510 18416 consensus_queue.cc:237] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:07.382373 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:07.382386 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:07.383325 17243 raft_consensus.cc:1203] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 01:59:07.383368 17242 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.384078 18863 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.388018 16974 raft_consensus.cc:1203] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:07.389604 15061 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.390158 18863 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.390308 17106 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.390693 18863 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.393165 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:07.406498 18071 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.407065 17106 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.409557 17243 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.411715 15309 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:07.412187 14912 catalog_manager.cc:5484] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7 reported cstate change: config changed from index 6 to 7, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.420289 15328 consensus_queue.cc:237] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:07.421671 17243 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.422310 18829 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.424283 17104 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.425606 18829 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.425877 18367 raft_consensus.cc:1050] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 to VOTER
I20240502 01:59:07.426410 18367 consensus_queue.cc:237] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:07.428507 16840 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.429132 17471 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.430953 15328 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.431208 19220 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:07.431380 17471 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.433132 15057 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.433482 17471 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.435269 17103 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.436607 18829 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.437060 17471 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.438721 18839 raft_consensus.cc:1013] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 84987ad9e4a54d96af8569e4aaf0ed64: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:07.439203 19220 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:07.439236 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ce85785e670a4264b44797bb27cb4ecb with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.439544 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:07.440117 19220 ts_tablet_manager.cc:1367] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.236s user 0.026s sys 0.004s
I20240502 01:59:07.440021 17243 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.440552 19220 raft_consensus.cc:348] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:07.440666 19220 raft_consensus.cc:729] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:07.440745 19220 consensus_queue.cc:260] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 1.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:07.441020 19220 ts_tablet_manager.cc:1398] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:07.441596 15488 tablet_copy_service.cc:342] P 7b66a8940fe54e21bec1b2db25b2fed5: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-3f938c5af51a43c9a23b753c4991e41d received from {username='slave'} at 127.7.221.69:53377
I20240502 01:59:07.441699 15488 tablet_copy_service.cc:433] P 7b66a8940fe54e21bec1b2db25b2fed5: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-3f938c5af51a43c9a23b753c4991e41d on tablet 3f938c5af51a43c9a23b753c4991e41d with peer 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:07.442514 14909 catalog_manager.cc:5484] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:07.442790 17103 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.447497 15040 tablet_service.cc:1514] Processing DeleteTablet for tablet ce85785e670a4264b44797bb27cb4ecb with delete_type TABLET_DATA_TOMBSTONED (TS ddae2a636423489693c12b7d7d166c09 not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:53384
I20240502 01:59:07.447510 15328 consensus_queue.cc:237] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:07.447705 19132 tablet_replica.cc:330] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09: stopping tablet replica
I20240502 01:59:07.447851 19132 raft_consensus.cc:2229] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.448015 19132 pending_rounds.cc:62] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09: Trying to abort 1 pending ops.
I20240502 01:59:07.448050 19132 pending_rounds.cc:68] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09: Aborting op as it isn't in flight: id { term: 1 index: 7 } timestamp: 7023063643673518080 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "ce85785e670a4264b44797bb27cb4ecb" old_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } } new_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } } }
I20240502 01:59:07.448210 19132 raft_consensus.cc:2894] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Aborting config change with OpId 1.7: Aborted: Op aborted
I20240502 01:59:07.448276 19132 raft_consensus.cc:2866] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 01:59:07.448312 19132 raft_consensus.cc:2258] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.448735 17243 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.449069 18416 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.449524 19132 ts_tablet_manager.cc:1875] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.450534 18416 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.451308 17103 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.451550 19132 ts_tablet_manager.cc:1888] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:07.451607 19132 log.cc:1196] T ce85785e670a4264b44797bb27cb4ecb P ddae2a636423489693c12b7d7d166c09: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/ce85785e670a4264b44797bb27cb4ecb
I20240502 01:59:07.452320 18880 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.452714 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ce85785e670a4264b44797bb27cb4ecb with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.452971 17243 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.453569 17103 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.453580 14912 catalog_manager.cc:5484] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 8 to 9, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:07.457403 15444 tablet_service.cc:1514] Processing DeleteTablet for tablet ce85785e670a4264b44797bb27cb4ecb with delete_type TABLET_DATA_TOMBSTONED (TS 7b66a8940fe54e21bec1b2db25b2fed5 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:60890
I20240502 01:59:07.457634 18999 tablet_replica.cc:330] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5: stopping tablet replica
I20240502 01:59:07.457763 18999 raft_consensus.cc:2229] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.457904 18999 pending_rounds.cc:62] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5: Trying to abort 1 pending ops.
I20240502 01:59:07.457926 18999 pending_rounds.cc:68] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5: Aborting op as it isn't in flight: id { term: 1 index: 6 } timestamp: 7023063643459502080 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "ce85785e670a4264b44797bb27cb4ecb" old_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } } new_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } } }
I20240502 01:59:07.458068 18999 raft_consensus.cc:2894] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Aborting config change with OpId 1.6: Aborted: Op aborted
I20240502 01:59:07.458118 18999 raft_consensus.cc:2866] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 01:59:07.458151 18999 raft_consensus.cc:2258] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.458381 14901 catalog_manager.cc:4830] TS ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123): tablet ce85785e670a4264b44797bb27cb4ecb (table rebalance_test_table_4 [id=0fd3b28283df457db390a2882ebcfef5]) successfully deleted
I20240502 01:59:07.460201 17471 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.462189 18818 tablet_copy_client.cc:523] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:07.463229 15328 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.463642 18818 tablet_bootstrap.cc:492] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap starting.
I20240502 01:59:07.465515 16840 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.465689 14906 catalog_manager.cc:5484] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
W20240502 01:59:07.467020 19318 consensus_queue.cc:1159] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" status { last_received { term: 1 index: 6 } last_committed_idx: 5 last_received_current_leader { term: 1 index: 6 } }
I20240502 01:59:07.468765 18999 ts_tablet_manager.cc:1875] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.470176 18839 raft_consensus.cc:1050] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
I20240502 01:59:07.470606 18839 consensus_queue.cc:237] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:07.470762 18999 ts_tablet_manager.cc:1888] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 01:59:07.470844 18999 log.cc:1196] T ce85785e670a4264b44797bb27cb4ecb P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/wals/ce85785e670a4264b44797bb27cb4ecb
I20240502 01:59:07.471029 19075 raft_consensus.cc:1050] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 to VOTER
I20240502 01:59:07.471414 19075 consensus_queue.cc:237] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.471972 14901 catalog_manager.cc:4830] TS 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): tablet ce85785e670a4264b44797bb27cb4ecb (table rebalance_test_table_4 [id=0fd3b28283df457db390a2882ebcfef5]) successfully deleted
I20240502 01:59:07.472213 15328 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.472559 19214 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.472715 17243 raft_consensus.cc:1261] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.472800 17105 raft_consensus.cc:1261] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.473197 18296 consensus_queue.cc:1035] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.473373 18767 consensus_queue.cc:1035] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.473578 15328 raft_consensus.cc:1261] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.474009 18767 consensus_queue.cc:1035] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.475476 15057 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.476261 18367 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.476812 15464 consensus_queue.cc:237] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:07.476787 15061 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
W20240502 01:59:07.478276 15396 consensus_peers.cc:485] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bf73e677bcd049869cadf9f8c9cd3434. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:07.478475 16974 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.478864 18296 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.481178 15194 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.481810 15057 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.481822 19034 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.482214 19034 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.483932 19073 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:07.484392 15057 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:07.487151 14906 catalog_manager.cc:5484] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 5 to 6, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:07.487886 16974 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:07.490227 15194 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:07.490942 15464 consensus_queue.cc:237] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:07.492435 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:07.493261 15444 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:07.493302 15194 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.494268 16840 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.494583 19034 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.494699 18367 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.499737 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:07.505065 15057 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.505503 18296 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.505852 16974 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.510021 17471 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.512336 15329 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.512507 19034 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.515035 14912 catalog_manager.cc:5484] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.515686 15057 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.519814 16840 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.522845 15194 consensus_queue.cc:237] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:07.524365 15329 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.524797 19214 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.528195 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:07.528235 15057 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.531163 19034 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.533106 17105 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.533452 15329 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.535382 14912 catalog_manager.cc:5484] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 7 to 8, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:07.535995 18296 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.536260 17471 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.537586 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet bf73e677bcd049869cadf9f8c9cd3434 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.537592 16840 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.538435 16974 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.540160 14906 catalog_manager.cc:5484] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 6 to 7, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:07.543936 15057 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.546631 15194 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.548156 18367 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.551071 17087 tablet_service.cc:1514] Processing DeleteTablet for tablet bf73e677bcd049869cadf9f8c9cd3434 with delete_type TABLET_DATA_TOMBSTONED (TS fc067ee690bb457e8109ad8e4eb43ab8 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:42304
I20240502 01:59:07.551128 18367 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.551571 16840 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.554296 14912 catalog_manager.cc:5484] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 reported cstate change: config changed from index 7 to 8, VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.556625 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:07.557209 17087 tablet_service.cc:1514] Processing DeleteTablet for tablet 6c4e85e7050b4dc8a4ce6bba818854a2 with delete_type TABLET_DATA_TOMBSTONED (TS fc067ee690bb457e8109ad8e4eb43ab8 not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:42304
I20240502 01:59:07.557436 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6c4e85e7050b4dc8a4ce6bba818854a2 with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.557534 18818 tablet_bootstrap.cc:492] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:07.557809 18818 tablet_bootstrap.cc:492] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap complete.
I20240502 01:59:07.557986 18818 ts_tablet_manager.cc:1367] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8: Time spent bootstrapping tablet: real 0.094s user 0.025s sys 0.000s
I20240502 01:59:07.558008 15328 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.558437 18818 raft_consensus.cc:348] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:07.558542 18818 raft_consensus.cc:729] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fc067ee690bb457e8109ad8e4eb43ab8, State: Initialized, Role: LEARNER
I20240502 01:59:07.558653 18818 consensus_queue.cc:260] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:07.558956 18818 ts_tablet_manager.cc:1398] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8: Time spent starting tablet: real 0.001s user 0.000s sys 0.002s
I20240502 01:59:07.559783 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-6db29f2516e1486782be0312c9bcac8b received from {username='slave'} at 127.7.221.71:41349
I20240502 01:59:07.559860 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-6db29f2516e1486782be0312c9bcac8b on tablet 6db29f2516e1486782be0312c9bcac8b with peer fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:07.566874 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:07.568557 15057 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.579953 16838 raft_consensus.cc:1203] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.5->[1.6-1.6] Dedup: 1.6->[]
I20240502 01:59:07.588776 17108 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.591696 17243 raft_consensus.cc:2932] T 986e7c97e0734e958f364dc5d1167846 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.593791 19363 ts_tablet_manager.cc:902] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:07.594339 19363 tablet_copy_client.cc:308] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 01:59:07.594857 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet 490e37fac0844a08a07a4b281bc33bf9 from peer d4e5422f66454511ae5a50ec6ab134ad ({username='slave'} at 127.7.221.70:33955)
I20240502 01:59:07.594965 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet 490e37fac0844a08a07a4b281bc33bf9 from peer d4e5422f66454511ae5a50ec6ab134ad at {username='slave'} at 127.7.221.70:33955: session id = d4e5422f66454511ae5a50ec6ab134ad-490e37fac0844a08a07a4b281bc33bf9
I20240502 01:59:07.595927 15351 tablet_copy_source_session.cc:205] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:07.597158 15444 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:07.601656 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:07.601991 19363 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 490e37fac0844a08a07a4b281bc33bf9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:07.604038 19363 tablet_copy_client.cc:783] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:07.604173 19363 tablet_copy_client.cc:647] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:07.607722 19363 tablet_copy_client.cc:523] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:07.608901 19363 tablet_bootstrap.cc:492] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap starting.
W20240502 01:59:07.612335 14898 catalog_manager.cc:4797] TS fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): delete failed for tablet bf73e677bcd049869cadf9f8c9cd3434 because the tablet was not found. No further retry: Not found: Tablet not found: bf73e677bcd049869cadf9f8c9cd3434
I20240502 01:59:07.615061 19362 tablet_replica.cc:330] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: stopping tablet replica
I20240502 01:59:07.616214 19362 raft_consensus.cc:2229] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.616832 19362 pending_rounds.cc:62] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: Trying to abort 1 pending ops.
I20240502 01:59:07.618913 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:07.619143 19362 pending_rounds.cc:68] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: Aborting op as it isn't in flight: id { term: 1 index: 6 } timestamp: 7023063643857113088 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "6c4e85e7050b4dc8a4ce6bba818854a2" old_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } } new_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } } }
I20240502 01:59:07.619531 19362 raft_consensus.cc:2894] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Aborting config change with OpId 1.6: Aborted: Op aborted
I20240502 01:59:07.619650 19362 raft_consensus.cc:2866] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 01:59:07.619737 19362 raft_consensus.cc:2258] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Raft consensus is shut down!
I20240502 01:59:07.622640 19362 ts_tablet_manager.cc:1875] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.625502 19362 ts_tablet_manager.cc:1888] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 01:59:07.625648 19362 log.cc:1196] T 6c4e85e7050b4dc8a4ce6bba818854a2 P fc067ee690bb457e8109ad8e4eb43ab8: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-6/wal/wals/6c4e85e7050b4dc8a4ce6bba818854a2
I20240502 01:59:07.626199 14898 catalog_manager.cc:4830] TS fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): tablet 6c4e85e7050b4dc8a4ce6bba818854a2 (table rebalance_test_table_9 [id=12ee6dc62f814bd7b3c13bee4495d9e0]) successfully deleted
I20240502 01:59:07.630548 15309 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:07.639922 15194 consensus_queue.cc:237] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:07.641592 17243 raft_consensus.cc:1261] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:07.641691 15464 raft_consensus.cc:1261] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:07.642071 18109 consensus_queue.cc:1035] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:07.642246 15057 raft_consensus.cc:1261] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:07.642637 18109 consensus_queue.cc:1035] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:07.643900 18109 consensus_queue.cc:1035] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:07.644186 19363 tablet_bootstrap.cc:492] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:07.644479 19363 tablet_bootstrap.cc:492] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap complete.
I20240502 01:59:07.644647 19363 ts_tablet_manager.cc:1367] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad: Time spent bootstrapping tablet: real 0.036s user 0.023s sys 0.000s
W20240502 01:59:07.644716 15131 consensus_peers.cc:485] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f97c5c9bb4c84957857efc76d1c7717f. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:07.645035 19363 raft_consensus.cc:348] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:07.645154 19363 raft_consensus.cc:729] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d4e5422f66454511ae5a50ec6ab134ad, State: Initialized, Role: LEARNER
I20240502 01:59:07.645246 19363 consensus_queue.cc:260] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:07.645478 19363 ts_tablet_manager.cc:1398] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:07.646427 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session d4e5422f66454511ae5a50ec6ab134ad-490e37fac0844a08a07a4b281bc33bf9 received from {username='slave'} at 127.7.221.70:33955
I20240502 01:59:07.646504 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session d4e5422f66454511ae5a50ec6ab134ad-490e37fac0844a08a07a4b281bc33bf9 on tablet 490e37fac0844a08a07a4b281bc33bf9 with peer d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:07.648177 18109 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.648528 15194 consensus_queue.cc:237] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:07.650557 15057 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
W20240502 01:59:07.650733 15130 consensus_peers.cc:485] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 21e2e55d1a0d4b6eb18b4ade9468607f. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:07.650763 15465 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:07.651046 15464 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.653066 14906 catalog_manager.cc:5484] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.653194 18839 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:07.654217 15328 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 01:59:07.654572 17471 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 01:59:07.669389 18109 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.671635 15464 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.672142 14912 catalog_manager.cc:5484] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:07.672605 16840 raft_consensus.cc:1203] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 01:59:07.674059 15328 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.677749 17910 raft_consensus.cc:1050] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 to VOTER
I20240502 01:59:07.678105 17910 consensus_queue.cc:237] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:07.679548 15464 raft_consensus.cc:1261] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.679878 15057 raft_consensus.cc:1261] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.680285 17471 consensus_queue.cc:1035] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.682314 17471 consensus_queue.cc:1035] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.684635 17243 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.686712 17242 raft_consensus.cc:1261] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.687673 17471 consensus_queue.cc:1035] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20240502 01:59:07.689307 15131 consensus_peers.cc:485] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f97c5c9bb4c84957857efc76d1c7717f. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:07.690004 18189 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.690358 15464 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.692158 15057 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.696290 14912 catalog_manager.cc:5484] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.698985 17242 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.703176 15194 consensus_queue.cc:237] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
W20240502 01:59:07.704501 15131 consensus_peers.cc:485] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f97c5c9bb4c84957857efc76d1c7717f. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:07.705468 18880 raft_consensus.cc:1050] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171: attempting to promote NON_VOTER d4e5422f66454511ae5a50ec6ab134ad to VOTER
I20240502 01:59:07.705931 18880 consensus_queue.cc:237] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:07.706173 15061 raft_consensus.cc:1261] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.706578 17910 consensus_queue.cc:1035] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.707933 16840 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.708061 15465 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.708344 18863 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.708446 16974 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.708725 18863 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.708884 18071 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.708935 15464 raft_consensus.cc:1261] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.709441 17910 consensus_queue.cc:1035] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.711256 17471 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.712229 15061 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.712285 15057 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.712956 18071 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.715848 18071 raft_consensus.cc:1013] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: attempt to promote peer 84987ad9e4a54d96af8569e4aaf0ed64: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:07.716212 14906 catalog_manager.cc:5484] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 6 to 7, VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.716743 15465 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.718654 17224 tablet_service.cc:1514] Processing DeleteTablet for tablet f97c5c9bb4c84957857efc76d1c7717f with delete_type TABLET_DATA_TOMBSTONED (TS 3b8d615c71e44ff887e0f037c3efcff7 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:37116
I20240502 01:59:07.720307 15194 consensus_queue.cc:237] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:07.720492 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f97c5c9bb4c84957857efc76d1c7717f with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
W20240502 01:59:07.721822 15131 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d24986c2ebab46f086e3f1f5a53af26b. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:07.722029 16840 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:07.722497 18367 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:07.725189 19214 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:07.725749 17108 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:07.726271 17910 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:07.727488 17108 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:07.727772 15328 consensus_queue.cc:237] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:07.728216 18863 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.729676 19414 tablet_replica.cc:330] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7: stopping tablet replica
I20240502 01:59:07.730422 14906 catalog_manager.cc:5484] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 9 to 10, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:07.731174 15464 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.731465 15061 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.731650 19414 raft_consensus.cc:2229] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.731819 19414 raft_consensus.cc:2258] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.733060 16974 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.733453 16972 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.735916 19414 ts_tablet_manager.cc:1875] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.736339 15194 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.737675 14906 catalog_manager.cc:5484] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: HEALTHY } } }
I20240502 01:59:07.737818 18880 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.737707 16838 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:07.738080 18880 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.738761 19414 ts_tablet_manager.cc:1888] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 01:59:07.738838 19414 log.cc:1196] T f97c5c9bb4c84957857efc76d1c7717f P 3b8d615c71e44ff887e0f037c3efcff7: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-7/wal/wals/f97c5c9bb4c84957857efc76d1c7717f
I20240502 01:59:07.738693 15464 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.739403 14902 catalog_manager.cc:4830] TS 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): tablet f97c5c9bb4c84957857efc76d1c7717f (table rebalance_test_table_4 [id=0fd3b28283df457db390a2882ebcfef5]) successfully deleted
I20240502 01:59:07.740510 18071 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.743659 15061 consensus_queue.cc:237] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:07.744036 15328 consensus_queue.cc:237] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:07.745348 16838 raft_consensus.cc:1261] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.745779 18624 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.746280 15195 raft_consensus.cc:1261] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.747401 18311 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.747781 16840 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:07.749027 18311 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.750375 15464 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.750437 16974 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.751411 15328 raft_consensus.cc:1261] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.752671 15061 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.753278 14912 catalog_manager.cc:5484] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 5 to 6. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:07.753415 15195 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.756609 18071 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.756835 18417 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.756808 15061 consensus_queue.cc:237] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:07.756904 16840 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.757109 18417 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.757109 18071 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.758775 16840 raft_consensus.cc:1261] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.759119 15195 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.759459 18386 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 01:59:07.759606 14994 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8ddc239a297a46e48e069b985cb7b350. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:07.759987 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 30620f4774b8414eb4e27bd5ecb6ecee with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.761422 15195 raft_consensus.cc:1261] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.761803 14913 catalog_manager.cc:5484] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 8 to 9, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.761970 18386 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.763741 16972 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.764343 15328 raft_consensus.cc:1261] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.765789 17224 tablet_service.cc:1514] Processing DeleteTablet for tablet 30620f4774b8414eb4e27bd5ecb6ecee with delete_type TABLET_DATA_TOMBSTONED (TS 3b8d615c71e44ff887e0f037c3efcff7 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:37116
I20240502 01:59:07.766536 14902 catalog_manager.cc:4830] TS 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): tablet 30620f4774b8414eb4e27bd5ecb6ecee (table rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]) successfully deleted
I20240502 01:59:07.767017 18386 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.767246 15464 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.767796 15328 consensus_queue.cc:237] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:07.768631 18296 raft_consensus.cc:1050] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 to VOTER
I20240502 01:59:07.769088 18296 consensus_queue.cc:237] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.769181 15194 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:07.769558 18071 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:07.770473 16972 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.770491 16974 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:07.771003 18071 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:07.771090 18766 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.771338 15328 raft_consensus.cc:858] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }.
Pending config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }:
New config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
W20240502 01:59:07.771394 15328 raft_consensus.cc:1565] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Could not prepare op 'term: 1 index: 7' and following 0 ops. Status for this op: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }.
Pending config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }:
New config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:07.771493 18071 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.771878 17242 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.772111 15328 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:07.772548 19034 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.773226 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 30620f4774b8414eb4e27bd5ecb6ecee with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.773278 15194 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.773896 15057 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.774291 19034 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.775411 15194 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.775851 19034 raft_consensus.cc:1013] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: attempt to promote peer d4e5422f66454511ae5a50ec6ab134ad: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:07.776101 18766 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.776166 14912 catalog_manager.cc:5484] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 9 to 10, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.777316 18880 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.778657 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 091d27d51acc4db0a1c7d20deeff5c10 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.779480 15444 tablet_service.cc:1514] Processing DeleteTablet for tablet 30620f4774b8414eb4e27bd5ecb6ecee with delete_type TABLET_DATA_TOMBSTONED (TS 7b66a8940fe54e21bec1b2db25b2fed5 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:60890
I20240502 01:59:07.779640 18999 tablet_replica.cc:330] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5: stopping tablet replica
I20240502 01:59:07.779758 18999 raft_consensus.cc:2229] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.779986 18999 raft_consensus.cc:2258] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.779867 16972 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.780861 18999 ts_tablet_manager.cc:1875] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.780709 16974 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.782721 15061 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.782933 15464 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.783104 18999 ts_tablet_manager.cc:1888] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 01:59:07.783159 18999 log.cc:1196] T 30620f4774b8414eb4e27bd5ecb6ecee P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/wals/30620f4774b8414eb4e27bd5ecb6ecee
I20240502 01:59:07.783730 18766 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.785624 17242 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.785710 18150 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.786046 16974 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.786417 15194 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.788005 19073 raft_consensus.cc:1050] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER d4e5422f66454511ae5a50ec6ab134ad to VOTER
I20240502 01:59:07.788079 14901 catalog_manager.cc:4830] TS 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): tablet 30620f4774b8414eb4e27bd5ecb6ecee (table rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]) successfully deleted
I20240502 01:59:07.788273 14914 catalog_manager.cc:5484] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 6 to 7, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:07.788368 19073 consensus_queue.cc:237] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.790083 14915 catalog_manager.cc:5484] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 reported cstate change: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.790323 16840 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.791186 17242 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.791564 19075 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.791981 15328 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.792135 16972 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.792510 19034 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.794100 15194 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.794517 18766 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.795578 16820 tablet_service.cc:1514] Processing DeleteTablet for tablet 091d27d51acc4db0a1c7d20deeff5c10 with delete_type TABLET_DATA_TOMBSTONED (TS 84987ad9e4a54d96af8569e4aaf0ed64 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:55224
I20240502 01:59:07.796509 16840 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.793890 15195 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.796854 18766 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.797081 19034 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.798532 14901 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 8ddc239a297a46e48e069b985cb7b350 with cas_config_opid_index 6: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 01:59:07.798921 16840 raft_consensus.cc:858] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }.
Pending config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }:
New config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.799026 17242 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.799113 16838 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
W20240502 01:59:07.799280 16840 raft_consensus.cc:1565] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Could not prepare op 'term: 1 index: 8' and following 0 ops. Status for this op: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }.
Pending config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }:
New config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.799604 16840 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.799795 14913 catalog_manager.cc:5484] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 6 to 7, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:07.800479 19456 tablet_replica.cc:330] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: stopping tablet replica
I20240502 01:59:07.800648 19456 raft_consensus.cc:2229] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Raft consensus shutting down.
I20240502 01:59:07.800909 19456 raft_consensus.cc:2258] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Raft consensus is shut down!
I20240502 01:59:07.801277 16974 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.802227 14910 catalog_manager.cc:5484] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 7 to 8, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:07.804577 15195 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.806998 15328 consensus_queue.cc:237] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:07.807379 15060 consensus_queue.cc:237] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:07.808971 15195 raft_consensus.cc:1261] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 01:59:07.809130 14994 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8ddc239a297a46e48e069b985cb7b350. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:07.809490 18907 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.809581 16838 raft_consensus.cc:1261] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.809984 18907 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.811406 15464 consensus_queue.cc:237] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.812338 16972 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.812795 18863 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.813036 17242 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.813314 16838 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.813742 18296 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.813980 18296 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.814399 16972 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.814890 19456 ts_tablet_manager.cc:1875] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.815068 18766 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.814986 18829 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.816414 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 091d27d51acc4db0a1c7d20deeff5c10 with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.817085 18150 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.817615 19456 ts_tablet_manager.cc:1888] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 01:59:07.817670 19456 log.cc:1196] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/wals/091d27d51acc4db0a1c7d20deeff5c10
I20240502 01:59:07.818341 14902 catalog_manager.cc:4830] TS 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): tablet 091d27d51acc4db0a1c7d20deeff5c10 (table rebalance_test_table_3 [id=0b29d58fe9c74d1cb4345bb6cfbd0535]) successfully deleted
I20240502 01:59:07.819324 15060 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.819593 15194 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.819808 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 8ddc239a297a46e48e069b985cb7b350 with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.820175 16973 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.820339 16836 raft_consensus.cc:2932] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.820511 14915 catalog_manager.cc:5484] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 7 to 8, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:07.822140 18417 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.823205 16840 raft_consensus.cc:858] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }.
Pending config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }:
New config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.823246 14914 catalog_manager.cc:5484] T 091d27d51acc4db0a1c7d20deeff5c10 P d4e5422f66454511ae5a50ec6ab134ad reported cstate change: config changed from index 7 to 8, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
W20240502 01:59:07.823318 16840 raft_consensus.cc:1565] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Could not prepare op 'term: 1 index: 9' and following 0 ops. Status for this op: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }.
Pending config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }:
New config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.823930 16840 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Committing config change with OpId 1.8: config changed from index 7 to 8, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.823923 19073 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.824384 15060 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.825675 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3f938c5af51a43c9a23b753c4991e41d with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.826634 16973 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.829864 18766 raft_consensus.cc:1050] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
I20240502 01:59:07.830165 18766 consensus_queue.cc:237] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.830540 15444 tablet_service.cc:1514] Processing DeleteTablet for tablet 091d27d51acc4db0a1c7d20deeff5c10 with delete_type TABLET_DATA_TOMBSTONED (TS 7b66a8940fe54e21bec1b2db25b2fed5 not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:60890
I20240502 01:59:07.830441 17242 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.830749 18999 tablet_replica.cc:330] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5: stopping tablet replica
I20240502 01:59:07.826591 14912 catalog_manager.cc:5484] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 8 to 9, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:07.830889 18999 raft_consensus.cc:2229] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.831065 18999 raft_consensus.cc:2258] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.832283 18999 ts_tablet_manager.cc:1875] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.834123 18999 ts_tablet_manager.cc:1888] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:07.834177 18999 log.cc:1196] T 091d27d51acc4db0a1c7d20deeff5c10 P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/wals/091d27d51acc4db0a1c7d20deeff5c10
I20240502 01:59:07.834668 15309 tablet_service.cc:1514] Processing DeleteTablet for tablet 8ddc239a297a46e48e069b985cb7b350 with delete_type TABLET_DATA_TOMBSTONED (TS f77c2a969bde45b49ac1bbbe242ff171 not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:43766
I20240502 01:59:07.834872 18851 tablet_replica.cc:330] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171: stopping tablet replica
I20240502 01:59:07.834918 17243 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:07.834988 18851 raft_consensus.cc:2229] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.835119 18851 raft_consensus.cc:2258] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.835515 14901 catalog_manager.cc:4830] TS 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): tablet 091d27d51acc4db0a1c7d20deeff5c10 (table rebalance_test_table_3 [id=0b29d58fe9c74d1cb4345bb6cfbd0535]) successfully deleted
I20240502 01:59:07.835978 18296 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:07.836035 18851 ts_tablet_manager.cc:1875] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.838032 18851 ts_tablet_manager.cc:1888] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:07.838088 18851 log.cc:1196] T 8ddc239a297a46e48e069b985cb7b350 P f77c2a969bde45b49ac1bbbe242ff171: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/8ddc239a297a46e48e069b985cb7b350
I20240502 01:59:07.838236 15061 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.838511 15174 tablet_service.cc:1514] Processing DeleteTablet for tablet 3f938c5af51a43c9a23b753c4991e41d with delete_type TABLET_DATA_TOMBSTONED (TS 26a3d5ced97d4dcc919b80fde53f6a5b not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:37206
I20240502 01:59:07.838562 14900 catalog_manager.cc:4830] TS f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): tablet 8ddc239a297a46e48e069b985cb7b350 (table rebalance_test_table_1 [id=1c35c23ce11f4c249032c42d4c499e42]) successfully deleted
I20240502 01:59:07.839432 15059 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.840562 16836 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:07.840910 18296 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.841851 15059 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:07.841848 16838 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.843525 18296 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.844424 15059 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.845845 15059 raft_consensus.cc:858] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }.
Pending config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }:
New config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
W20240502 01:59:07.845903 15059 raft_consensus.cc:1565] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Could not prepare op 'term: 1 index: 9' and following 1 ops. Status for this op: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }.
Pending config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }:
New config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.846166 15059 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.849416 16973 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:07.849941 19034 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:07.850224 19476 tablet_replica.cc:330] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b: stopping tablet replica
I20240502 01:59:07.850405 19476 raft_consensus.cc:2229] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.850693 19476 raft_consensus.cc:2258] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.851485 19476 ts_tablet_manager.cc:1875] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.851922 15059 raft_consensus.cc:858] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }.
Pending config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }:
New config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
W20240502 01:59:07.852074 15059 raft_consensus.cc:1565] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Could not prepare op 'term: 1 index: 10' and following 0 ops. Status for this op: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }.
Pending config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }:
New config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.852347 15059 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.854236 17108 raft_consensus.cc:1203] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 01:59:07.855598 19034 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.857620 19476 ts_tablet_manager.cc:1888] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 01:59:07.858165 16972 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.858392 14914 catalog_manager.cc:5484] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 9 to 10, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:07.859827 17243 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.862020 15061 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.863263 15464 consensus_queue.cc:237] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:07.864789 17243 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:07.864894 16972 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:07.865244 18296 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:07.865408 15061 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:07.865520 18296 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:07.865777 19073 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:07.868258 18766 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.869562 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3f938c5af51a43c9a23b753c4991e41d with cas_config_opid_index 10: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.870069 17243 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.870529 14913 catalog_manager.cc:5484] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 10 to 11, VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:07.871727 16972 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.874904 15061 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:07.875337 16820 tablet_service.cc:1514] Processing DeleteTablet for tablet 3f938c5af51a43c9a23b753c4991e41d with delete_type TABLET_DATA_TOMBSTONED (TS 84987ad9e4a54d96af8569e4aaf0ed64 not found in new config with opid_index 11) from {username='slave'} at 127.0.0.1:55224
I20240502 01:59:07.875635 19456 tablet_replica.cc:330] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: stopping tablet replica
I20240502 01:59:07.875779 19456 raft_consensus.cc:2229] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Raft consensus shutting down.
I20240502 01:59:07.875934 19456 raft_consensus.cc:2258] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Raft consensus is shut down!
I20240502 01:59:07.881459 15328 consensus_queue.cc:237] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
W20240502 01:59:07.881742 19457 consensus_queue.cc:1159] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" status { last_received { term: 1 index: 9 } last_committed_idx: 9 last_received_current_leader { term: 1 index: 9 } }
I20240502 01:59:07.882975 16838 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:07.883458 18863 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:07.885339 19476 log.cc:1196] T 3f938c5af51a43c9a23b753c4991e41d P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/3f938c5af51a43c9a23b753c4991e41d
I20240502 01:59:07.862538 15195 consensus_queue.cc:237] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:07.886008 14900 catalog_manager.cc:4830] TS 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905): tablet 3f938c5af51a43c9a23b753c4991e41d (table rebalance_test_table_2 [id=73bf3d4779b34c7290b42f7f561e8e11]) successfully deleted
I20240502 01:59:07.874504 17910 raft_consensus.cc:1050] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 to VOTER
I20240502 01:59:07.886330 15061 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:07.886569 17910 consensus_queue.cc:237] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:07.886811 18863 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:07.888293 17108 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.877840 19214 raft_consensus.cc:1050] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER d4e5422f66454511ae5a50ec6ab134ad to VOTER
W20240502 01:59:07.888482 19214 raft_consensus.cc:1053] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b: Unable to promote non-voter d4e5422f66454511ae5a50ec6ab134ad: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }.
Pending config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
W20240502 01:59:07.888641 15265 consensus_peers.cc:485] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:07.888998 15061 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.889372 18109 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.889672 15328 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.889712 18109 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.890053 18109 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.892259 17243 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.892596 15464 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.892791 18109 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.892988 17471 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
W20240502 01:59:07.893326 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 5804597cc507483dba622e6ec44e5ead. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:07.893445 16972 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:07.893821 17471 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.897522 19214 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.897900 15328 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.898129 18109 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.899631 17243 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.900055 18839 raft_consensus.cc:1050] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER d4e5422f66454511ae5a50ec6ab134ad to VOTER
I20240502 01:59:07.900363 18839 consensus_queue.cc:237] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:07.901363 16838 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.902215 15061 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.902500 15328 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.902576 17910 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.902844 18109 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.903271 15061 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.903746 16973 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.903760 18863 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.904460 15057 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.905941 14906 catalog_manager.cc:5484] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.905936 16972 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.907100 17104 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.907691 18109 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.909029 17104 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.909312 14912 catalog_manager.cc:5484] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 reported cstate change: config changed from index 10 to 11, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.914144 18109 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:07.914422 18109 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.915484 14913 catalog_manager.cc:5484] T 5804597cc507483dba622e6ec44e5ead P 3b8d615c71e44ff887e0f037c3efcff7 reported cstate change: config changed from index 8 to 9, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.918081 15059 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.918682 15328 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.920238 14909 catalog_manager.cc:5484] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:07.920558 17104 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.922214 19456 ts_tablet_manager.cc:1875] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.923779 16972 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.924539 19456 ts_tablet_manager.cc:1888] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 01:59:07.924620 19456 log.cc:1196] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/wals/3f938c5af51a43c9a23b753c4991e41d
I20240502 01:59:07.925150 14902 catalog_manager.cc:4830] TS 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): tablet 3f938c5af51a43c9a23b753c4991e41d (table rebalance_test_table_2 [id=73bf3d4779b34c7290b42f7f561e8e11]) successfully deleted
I20240502 01:59:07.931878 15195 consensus_queue.cc:237] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:07.931989 15464 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:07.933351 17104 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.933746 15328 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.934327 17910 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.935210 19214 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.937475 19214 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.939354 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cd00409d47954ee2b7555e7ff09a6e28 with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.940073 17104 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.940362 15328 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.940594 14914 catalog_manager.cc:5484] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:07.943506 16972 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:07.944063 18839 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:07.944734 15040 tablet_service.cc:1514] Processing DeleteTablet for tablet cd00409d47954ee2b7555e7ff09a6e28 with delete_type TABLET_DATA_TOMBSTONED (TS ddae2a636423489693c12b7d7d166c09 not found in new config with opid_index 8) from {username='slave'} at 127.0.0.1:53384
I20240502 01:59:07.945323 16973 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 01:59:07.945986 15195 consensus_queue.cc:237] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:07.946708 19132 tablet_replica.cc:330] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09: stopping tablet replica
I20240502 01:59:07.946853 19132 raft_consensus.cc:2229] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:07.947068 19132 raft_consensus.cc:2258] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:07.947345 15059 consensus_queue.cc:237] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:07.948657 16973 raft_consensus.cc:1261] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.948740 15328 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
W20240502 01:59:07.948982 14995 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8f2fb504a49c44edac92fd935555c278. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:07.949003 17104 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:07.948987 18386 consensus_queue.cc:1035] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:07.949378 17910 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.949807 19132 ts_tablet_manager.cc:1875] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:07.949971 17910 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:07.950385 15195 raft_consensus.cc:1261] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:07.951915 19132 ts_tablet_manager.cc:1888] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.7
I20240502 01:59:07.951967 19132 log.cc:1196] T cd00409d47954ee2b7555e7ff09a6e28 P ddae2a636423489693c12b7d7d166c09: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/cd00409d47954ee2b7555e7ff09a6e28
I20240502 01:59:07.952349 19214 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.952659 14901 catalog_manager.cc:4830] TS ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123): tablet cd00409d47954ee2b7555e7ff09a6e28 (table rebalance_test_table_8 [id=bf7d5082701149bea6b9e5c480f53c10]) successfully deleted
I20240502 01:59:07.953076 17104 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.954547 15328 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.957309 14912 catalog_manager.cc:5484] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 8 to 9, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:07.958046 18386 consensus_queue.cc:1035] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
I20240502 01:59:07.963817 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cd00409d47954ee2b7555e7ff09a6e28 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:07.964924 16954 tablet_service.cc:1514] Processing DeleteTablet for tablet cd00409d47954ee2b7555e7ff09a6e28 with delete_type TABLET_DATA_TOMBSTONED (TS d4e5422f66454511ae5a50ec6ab134ad not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:52144
I20240502 01:59:07.966480 18386 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:07.969162 14913 catalog_manager.cc:5484] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 6 to 7, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:07.969336 16973 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:07.978097 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:07.981418 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:07.991942 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:07.997522 15444 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:08.012305 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.013953 15195 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:08.021451 15309 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:08.024071 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:08.041854 19537 tablet_replica.cc:330] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad: stopping tablet replica
I20240502 01:59:08.042052 19537 raft_consensus.cc:2229] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:08.042249 19537 raft_consensus.cc:2258] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:08.045825 19537 ts_tablet_manager.cc:1875] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:08.048579 19537 ts_tablet_manager.cc:1888] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 01:59:08.048753 19537 log.cc:1196] T cd00409d47954ee2b7555e7ff09a6e28 P d4e5422f66454511ae5a50ec6ab134ad: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal/wals/cd00409d47954ee2b7555e7ff09a6e28
I20240502 01:59:08.049407 14901 catalog_manager.cc:4830] TS d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): tablet cd00409d47954ee2b7555e7ff09a6e28 (table rebalance_test_table_8 [id=bf7d5082701149bea6b9e5c480f53c10]) successfully deleted
I20240502 01:59:08.057502 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:08.058218 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.071049 15328 consensus_queue.cc:237] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:08.072029 15444 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:08.072759 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:08.073148 15195 raft_consensus.cc:1261] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:08.074295 15464 raft_consensus.cc:1261] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:08.074920 16973 raft_consensus.cc:1261] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:08.075474 18880 consensus_queue.cc:1035] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:08.077255 18417 consensus_queue.cc:1035] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:08.077512 18417 consensus_queue.cc:1035] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 01:59:08.084105 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 490e37fac0844a08a07a4b281bc33bf9. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:08.085382 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.085399 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.085965 18417 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.090965 15195 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.095538 14913 catalog_manager.cc:5484] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:08.098984 15464 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.106017 15309 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:08.125806 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.128983 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.142969 19363 ts_tablet_manager.cc:902] T 21e2e55d1a0d4b6eb18b4ade9468607f P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:08.143128 18829 raft_consensus.cc:1050] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171: attempting to promote NON_VOTER d4e5422f66454511ae5a50ec6ab134ad to VOTER
I20240502 01:59:08.143479 19363 tablet_copy_client.cc:308] T 21e2e55d1a0d4b6eb18b4ade9468607f P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:08.143548 18829 consensus_queue.cc:237] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:08.144032 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet 21e2e55d1a0d4b6eb18b4ade9468607f from peer d4e5422f66454511ae5a50ec6ab134ad ({username='slave'} at 127.7.221.70:59193)
I20240502 01:59:08.144142 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet 21e2e55d1a0d4b6eb18b4ade9468607f from peer d4e5422f66454511ae5a50ec6ab134ad at {username='slave'} at 127.7.221.70:59193: session id = d4e5422f66454511ae5a50ec6ab134ad-21e2e55d1a0d4b6eb18b4ade9468607f
I20240502 01:59:08.145148 15216 tablet_copy_source_session.cc:205] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
W20240502 01:59:08.145248 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 490e37fac0844a08a07a4b281bc33bf9. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:08.145900 16973 raft_consensus.cc:1261] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:08.145980 15195 raft_consensus.cc:1261] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:08.146351 18417 consensus_queue.cc:1035] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:08.146468 16972 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.147279 19363 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 21e2e55d1a0d4b6eb18b4ade9468607f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:08.148545 18417 consensus_queue.cc:1035] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:08.149184 19363 tablet_copy_client.cc:783] T 21e2e55d1a0d4b6eb18b4ade9468607f P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:08.149338 19363 tablet_copy_client.cc:647] T 21e2e55d1a0d4b6eb18b4ade9468607f P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:08.153962 19363 tablet_copy_client.cc:523] T 21e2e55d1a0d4b6eb18b4ade9468607f P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:08.155213 19363 tablet_bootstrap.cc:492] T 21e2e55d1a0d4b6eb18b4ade9468607f P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap starting.
I20240502 01:59:08.155879 18880 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.156241 15464 raft_consensus.cc:1261] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:08.156893 16973 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.157290 18880 consensus_queue.cc:1035] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:08.158550 15195 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.158918 15464 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.159611 14910 catalog_manager.cc:5484] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 5 to 6, d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:08.221745 15195 consensus_queue.cc:237] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } }
I20240502 01:59:08.223248 16973 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:08.223768 17243 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:08.224604 17910 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:08.226668 18839 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
W20240502 01:59:08.226862 15129 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): Couldn't send request to peer f77c2a969bde45b49ac1bbbe242ff171. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 81c8fe21ec7545e79450532b43a61ee7. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:08.228338 18839 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } } }
I20240502 01:59:08.229480 17243 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } } }
I20240502 01:59:08.231552 16973 raft_consensus.cc:2932] T 81c8fe21ec7545e79450532b43a61ee7 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } } }
I20240502 01:59:08.233618 14910 catalog_manager.cc:5484] T 81c8fe21ec7545e79450532b43a61ee7 P 3b8d615c71e44ff887e0f037c3efcff7 reported cstate change: config changed from index 9 to 10, NON_VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } } }
I20240502 01:59:08.245101 19636 ts_tablet_manager.cc:902] T d24986c2ebab46f086e3f1f5a53af26b P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:08.245585 19636 tablet_copy_client.cc:308] T d24986c2ebab46f086e3f1f5a53af26b P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:08.246083 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet d24986c2ebab46f086e3f1f5a53af26b from peer 3b8d615c71e44ff887e0f037c3efcff7 ({username='slave'} at 127.7.221.72:57673)
I20240502 01:59:08.246197 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet d24986c2ebab46f086e3f1f5a53af26b from peer 3b8d615c71e44ff887e0f037c3efcff7 at {username='slave'} at 127.7.221.72:57673: session id = 3b8d615c71e44ff887e0f037c3efcff7-d24986c2ebab46f086e3f1f5a53af26b
I20240502 01:59:08.247299 15216 tablet_copy_source_session.cc:205] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:08.248629 19636 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d24986c2ebab46f086e3f1f5a53af26b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:08.252975 19636 tablet_copy_client.cc:783] T d24986c2ebab46f086e3f1f5a53af26b P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:08.256953 19636 tablet_copy_client.cc:647] T d24986c2ebab46f086e3f1f5a53af26b P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:08.260577 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.264873 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.269136 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:08.270982 15444 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:08.282125 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.282408 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:08.294572 19636 tablet_copy_client.cc:523] T d24986c2ebab46f086e3f1f5a53af26b P 3b8d615c71e44ff887e0f037c3efcff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:08.296314 19636 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap starting.
I20240502 01:59:08.305073 19659 ts_tablet_manager.cc:902] T 5804597cc507483dba622e6ec44e5ead P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:08.305784 19659 tablet_copy_client.cc:308] T 5804597cc507483dba622e6ec44e5ead P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:08.306385 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet 5804597cc507483dba622e6ec44e5ead from peer d4e5422f66454511ae5a50ec6ab134ad ({username='slave'} at 127.7.221.70:59193)
I20240502 01:59:08.306488 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet 5804597cc507483dba622e6ec44e5ead from peer d4e5422f66454511ae5a50ec6ab134ad at {username='slave'} at 127.7.221.70:59193: session id = d4e5422f66454511ae5a50ec6ab134ad-5804597cc507483dba622e6ec44e5ead
I20240502 01:59:08.307602 15216 tablet_copy_source_session.cc:205] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:08.310261 19659 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5804597cc507483dba622e6ec44e5ead. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:08.311388 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 01:59:08.313092 19659 tablet_copy_client.cc:783] T 5804597cc507483dba622e6ec44e5ead P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:08.313303 19659 tablet_copy_client.cc:647] T 5804597cc507483dba622e6ec44e5ead P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:08.317329 19659 tablet_copy_client.cc:523] T 5804597cc507483dba622e6ec44e5ead P d4e5422f66454511ae5a50ec6ab134ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:08.320943 19659 tablet_bootstrap.cc:492] T 5804597cc507483dba622e6ec44e5ead P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap starting.
I20240502 01:59:08.327119 15444 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:08.329191 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.331930 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.332139 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.337544 15464 consensus_queue.cc:237] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:08.339272 16973 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 01:59:08.339622 15059 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 01:59:08.340225 18279 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 01:59:08.340364 17243 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 01:59:08.340543 18279 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 01:59:08.340742 18767 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 01:59:08.341147 15309 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:08.345660 19675 ts_tablet_manager.cc:902] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:08.346135 19675 tablet_copy_client.cc:308] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:08.346706 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet f97c5c9bb4c84957857efc76d1c7717f from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:43177)
I20240502 01:59:08.346822 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet f97c5c9bb4c84957857efc76d1c7717f from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:43177: session id = 84987ad9e4a54d96af8569e4aaf0ed64-f97c5c9bb4c84957857efc76d1c7717f
I20240502 01:59:08.347736 15216 tablet_copy_source_session.cc:205] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:08.347772 18296 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:08.348330 16973 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:08.349892 17243 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:08.351871 14915 catalog_manager.cc:5484] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad reported cstate change: config changed from index 11 to 12. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:08.352986 15059 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:08.364064 19675 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f97c5c9bb4c84957857efc76d1c7717f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:08.364100 15464 consensus_queue.cc:237] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:08.365542 16973 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 01:59:08.365994 19075 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 01:59:08.369218 17243 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 01:59:08.369463 19075 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:08.369645 19034 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 01:59:08.369715 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:08.370537 17243 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:08.371318 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3f938c5af51a43c9a23b753c4991e41d with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:08.372495 16973 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:08.373328 14915 catalog_manager.cc:5484] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 reported cstate change: config changed from index 12 to 13, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 01:59:08.376204 15040 tablet_service.cc:1514] Processing DeleteTablet for tablet 3f938c5af51a43c9a23b753c4991e41d with delete_type TABLET_DATA_TOMBSTONED (TS ddae2a636423489693c12b7d7d166c09 not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:53384
I20240502 01:59:08.377822 19132 tablet_replica.cc:330] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09: stopping tablet replica
I20240502 01:59:08.377948 19132 raft_consensus.cc:2229] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:08.378180 19132 raft_consensus.cc:2258] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:08.378180 15464 consensus_queue.cc:237] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 1.13, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
W20240502 01:59:08.380685 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:08.381035 17243 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 01:59:08.381404 19132 ts_tablet_manager.cc:1875] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:08.382223 18279 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 01:59:08.382537 15059 consensus_queue.cc:237] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } }
I20240502 01:59:08.383562 19132 ts_tablet_manager.cc:1888] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240502 01:59:08.383627 19132 log.cc:1196] T 3f938c5af51a43c9a23b753c4991e41d P ddae2a636423489693c12b7d7d166c09: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/3f938c5af51a43c9a23b753c4991e41d
I20240502 01:59:08.384037 16838 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:08.384191 15195 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:08.384661 18624 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:08.384867 16972 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 01:59:08.385201 18624 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:08.387427 18279 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 01:59:08.387809 18279 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
W20240502 01:59:08.388129 14994 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): Couldn't send request to peer 7b66a8940fe54e21bec1b2db25b2fed5. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 77ae087802144c45924c1b79f561afe8. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:08.388249 17242 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:08.390797 14910 catalog_manager.cc:5484] T 3f938c5af51a43c9a23b753c4991e41d P 3b8d615c71e44ff887e0f037c3efcff7 reported cstate change: config changed from index 13 to 14, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:08.392226 16972 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:08.392666 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.394541 14901 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 3f938c5af51a43c9a23b753c4991e41d with cas_config_opid_index 13: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 01:59:08.395251 14901 catalog_manager.cc:4830] TS ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123): tablet 3f938c5af51a43c9a23b753c4991e41d (table rebalance_test_table_2 [id=73bf3d4779b34c7290b42f7f561e8e11]) successfully deleted
I20240502 01:59:08.400777 15309 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:08.404325 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.404773 19363 tablet_bootstrap.cc:492] T 21e2e55d1a0d4b6eb18b4ade9468607f P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:08.405150 19363 tablet_bootstrap.cc:492] T 21e2e55d1a0d4b6eb18b4ade9468607f P d4e5422f66454511ae5a50ec6ab134ad: Bootstrap complete.
I20240502 01:59:08.405433 19363 ts_tablet_manager.cc:1367] T 21e2e55d1a0d4b6eb18b4ade9468607f P d4e5422f66454511ae5a50ec6ab134ad: Time spent bootstrapping tablet: real 0.250s user 0.026s sys 0.004s
I20240502 01:59:08.406034 19363 raft_consensus.cc:348] T 21e2e55d1a0d4b6eb18b4ade9468607f P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:08.406252 19363 raft_consensus.cc:729] T 21e2e55d1a0d4b6eb18b4ade9468607f P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d4e5422f66454511ae5a50ec6ab134ad, State: Initialized, Role: LEARNER
I20240502 01:59:08.406419 19363 consensus_queue.cc:260] T 21e2e55d1a0d4b6eb18b4ade9468607f P d4e5422f66454511ae5a50ec6ab134ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:08.407936 19675 tablet_copy_client.cc:783] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:08.408159 19675 tablet_copy_client.cc:647] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:08.414824 19675 tablet_copy_client.cc:523] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:08.415745 18829 raft_consensus.cc:1050] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171: attempting to promote NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 to VOTER
I20240502 01:59:08.415714 18624 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } }
I20240502 01:59:08.416137 18829 consensus_queue.cc:237] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:08.416429 19675 tablet_bootstrap.cc:492] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
I20240502 01:59:08.414764 19699 ts_tablet_manager.cc:902] T 8ddc239a297a46e48e069b985cb7b350 P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:08.417656 15195 raft_consensus.cc:1261] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
F20240502 01:59:08.417493 19699 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-091d27d51acc4db0a1c7d20deeff5c10.queue_time_us, tablet-download-pool-091d27d51acc4db0a1c7d20deeff5c10.queue_time_us) (tablet-download-pool-091d27d51acc4db0a1c7d20deeff5c10.run_cpu_time_us, tablet-download-pool-091d27d51acc4db0a1c7d20deeff5c10.run_cpu_time_us) (tablet-download-pool-091d27d51acc4db0a1c7d20deeff5c10.run_wall_time_us, tablet-download-pool-091d27d51acc4db0a1c7d20deeff5c10.run_wall_time_us) (tablet-download-pool-21e2e55d1a0d4b6eb18b4ade9468607f.queue_time_us, tablet-download-pool-21e2e55d1a0d4b6eb18b4ade9468607f.queue_time_us) (tablet-download-pool-21e2e55d1a0d4b6eb18b4ade9468607f.run_cpu_time_us, tablet-download-pool-21e2e55d1a0d4b6eb18b4ade9468607f.run_cpu_time_us) (tablet-download-pool-21e2e55d1a0d4b6eb18b4ade9468607f.run_wall_time_us, tablet-download-pool-21e2e55d1a0d4b6eb18b4ade9468607f.run_wall_time_us) (tablet-download-pool-30620f4774b8414eb4e27bd5ecb6ecee.queue_time_us, tablet-download-pool-30620f4774b8414eb4e27bd5ecb6ecee.queue_time_us) (tablet-download-pool-30620f4774b8414eb4e27bd5ecb6ecee.run_cpu_time_us, tablet-download-pool-30620f4774b8414eb4e27bd5ecb6ecee.run_cpu_time_us) (tablet-download-pool-30620f4774b8414eb4e27bd5ecb6ecee.run_wall_time_us, tablet-download-pool-30620f4774b8414eb4e27bd5ecb6ecee.run_wall_time_us) (tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.queue_time_us, tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.queue_time_us) (tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.run_cpu_time_us, tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.run_cpu_time_us) (tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.run_wall_time_us, tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.run_wall_time_us) (tablet-download-pool-490e37fac0844a08a07a4b281bc33bf9.queue_time_us, tablet-download-pool-490e37fac0844a08a07a4b281bc33bf9.queue_time_us) (tablet-download-pool-490e37fac0844a08a07a4b281bc33bf9.run_cpu_time_us, tablet-download-pool-490e37fac0844a08a07a4b281bc33bf9.run_cpu_time_us) (tablet-download-pool-490e37fac0844a08a07a4b281bc33bf9.run_wall_time_us, tablet-download-pool-490e37fac0844a08a07a4b281bc33bf9.run_wall_time_us) (tablet-download-pool-5804597cc507483dba622e6ec44e5ead.queue_time_us, tablet-download-pool-5804597cc507483dba622e6ec44e5ead.queue_time_us) (tablet-download-pool-5804597cc507483dba622e6ec44e5ead.run_cpu_time_us, tablet-download-pool-5804597cc507483dba622e6ec44e5ead.run_cpu_time_us) (tablet-download-pool-5804597cc507483dba622e6ec44e5ead.run_wall_time_us, tablet-download-pool-5804597cc507483dba622e6ec44e5ead.run_wall_time_us) (tablet-download-pool-81c8fe21ec7545e79450532b43a61ee7.queue_time_us, tablet-download-pool-81c8fe21ec7545e79450532b43a61ee7.queue_time_us) (tablet-download-pool-81c8fe21ec7545e79450532b43a61ee7.run_cpu_time_us, tablet-download-pool-81c8fe21ec7545e79450532b43a61ee7.run_cpu_time_us) (tablet-download-pool-81c8fe21ec7545e79450532b43a61ee7.run_wall_time_us, tablet-download-pool-81c8fe21ec7545e79450532b43a61ee7.run_wall_time_us) (tablet-download-pool-8ddc239a297a46e48e069b985cb7b350.queue_time_us, tablet-download-pool-8ddc239a297a46e48e069b985cb7b350.queue_time_us) (tablet-download-pool-8f2fb504a49c44edac92fd935555c278.queue_time_us, tablet-download-pool-8f2fb504a49c44edac92fd935555c278.queue_time_us) (tablet-download-pool-8f2fb504a49c44edac92fd935555c278.run_cpu_time_us, tablet-download-pool-8f2fb504a49c44edac92fd935555c278.run_cpu_time_us) (tablet-download-pool-8f2fb504a49c44edac92fd935555c278.run_wall_time_us, tablet-download-pool-8f2fb504a49c44edac92fd935555c278.run_wall_time_us) (tablet-download-pool-a901eb3063ea4911a822552db01d365d.queue_time_us, tablet-download-pool-a901eb3063ea4911a822552db01d365d.queue_time_us) (tablet-download-pool-a901eb3063ea4911a822552db01d365d.run_cpu_time_us, tablet-download-pool-a901eb3063ea4911a822552db01d365d.run_cpu_time_us) (tablet-download-pool-a901eb3063ea4911a822552db01d365d.run_wall_time_us, tablet-download-pool-a901eb3063ea4911a822552db01d365d.run_wall_time_us) (tablet-download-pool-bf73e677bcd049869cadf9f8c9cd3434.queue_time_us, tablet-download-pool-bf73e677bcd049869cadf9f8c9cd3434.queue_time_us) (tablet-download-pool-bf73e677bcd049869cadf9f8c9cd3434.run_cpu_time_us, tablet-download-pool-bf73e677bcd049869cadf9f8c9cd3434.run_cpu_time_us) (tablet-download-pool-bf73e677bcd049869cadf9f8c9cd3434.run_wall_time_us, tablet-download-pool-bf73e677bcd049869cadf9f8c9cd3434.run_wall_time_us) (tablet-download-pool-cd00409d47954ee2b7555e7ff09a6e28.queue_time_us, tablet-download-pool-cd00409d47954ee2b7555e7ff09a6e28.queue_time_us) (tablet-download-pool-cd00409d47954ee2b7555e7ff09a6e28.run_cpu_time_us, tablet-download-pool-cd00409d47954ee2b7555e7ff09a6e28.run_cpu_time_us) (tablet-download-pool-cd00409d47954ee2b7555e7ff09a6e28.run_wall_time_us, tablet-download-pool-cd00409d47954ee2b7555e7ff09a6e28.run_wall_time_us) (tablet-download-pool-d1c832c721fb4b15bc6d266a1c295401.queue_time_us, tablet-download-pool-d1c832c721fb4b15bc6d266a1c295401.queue_time_us) (tablet-download-pool-d1c832c721fb4b15bc6d266a1c295401.run_cpu_time_us, tablet-download-pool-d1c832c721fb4b15bc6d266a1c295401.run_cpu_time_us) (tablet-download-pool-d1c832c721fb4b15bc6d266a1c295401.run_wall_time_us, tablet-download-pool-d1c832c721fb4b15bc6d266a1c295401.run_wall_time_us) (tablet-download-pool-e2f4ca025890492880608c7975650969.queue_time_us, tablet-download-pool-e2f4ca025890492880608c7975650969.queue_time_us) (tablet-download-pool-e2f4ca025890492880608c7975650969.run_cpu_time_us, tablet-download-pool-e2f4ca025890492880608c7975650969.run_cpu_time_us) (tablet-download-pool-e2f4ca025890492880608c7975650969.run_wall_time_us, tablet-download-pool-e2f4ca025890492880608c7975650969.run_wall_time_us) (tablet-download-pool-fc07e430e3e74e3885f263881bd04976.queue_time_us, tablet-download-pool-fc07e430e3e74e3885f263881bd04976.queue_time_us) (tablet-download-pool-fc07e430e3e74e3885f263881bd04976.run_cpu_time_us, tablet-download-pool-fc07e430e3e74e3885f263881bd04976.run_cpu_time_us) (tablet-download-pool-fc07e430e3e74e3885f263881bd04976.run_wall_time_us, tablet-download-pool-fc07e430e3e74e3885f263881bd04976.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
I20240502 01:59:08.418675 18880 consensus_queue.cc:1035] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:08.419124 15059 raft_consensus.cc:1261] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:08.419644 16838 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } }
@ 0x7f8549d8cdcd google::LogMessage::Fail() at ??:0
I20240502 01:59:08.421788 18880 consensus_queue.cc:1035] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:08.422119 19363 ts_tablet_manager.cc:1398] T 21e2e55d1a0d4b6eb18b4ade9468607f P d4e5422f66454511ae5a50ec6ab134ad: Time spent starting tablet: real 0.017s user 0.000s sys 0.001s
I20240502 01:59:08.422524 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session d4e5422f66454511ae5a50ec6ab134ad-21e2e55d1a0d4b6eb18b4ade9468607f received from {username='slave'} at 127.7.221.70:59193
I20240502 01:59:08.422621 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session d4e5422f66454511ae5a50ec6ab134ad-21e2e55d1a0d4b6eb18b4ade9468607f on tablet 21e2e55d1a0d4b6eb18b4ade9468607f with peer d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:08.423836 14915 catalog_manager.cc:5484] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 reported cstate change: config changed from index 6 to 7, NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } }
I20240502 01:59:08.425225 15194 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } }
I20240502 01:59:08.430457 17104 raft_consensus.cc:1261] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:08.431018 18880 consensus_queue.cc:1035] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:08.448752 15194 consensus_queue.cc:237] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } }
W20240502 01:59:08.450762 15130 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): Couldn't send request to peer 7b66a8940fe54e21bec1b2db25b2fed5. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cd00409d47954ee2b7555e7ff09a6e28. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:08.450912 17104 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:08.451331 15328 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:08.451999 17910 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:08.452844 17910 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
@ 0x7f8549d90b93 google::LogMessage::SendToLog() at ??:0
I20240502 01:59:08.455082 17471 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } }
I20240502 01:59:08.457336 15328 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } }
I20240502 01:59:08.457901 14910 catalog_manager.cc:5484] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 9 to 10, NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:08.461422 17104 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } }
I20240502 01:59:08.465026 19636 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=10 overwritten=0 applied=10 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:08.465338 19636 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P 3b8d615c71e44ff887e0f037c3efcff7: Bootstrap complete.
I20240502 01:59:08.465492 19636 ts_tablet_manager.cc:1367] T d24986c2ebab46f086e3f1f5a53af26b P 3b8d615c71e44ff887e0f037c3efcff7: Time spent bootstrapping tablet: real 0.169s user 0.024s sys 0.002s
I20240502 01:59:08.465906 19636 raft_consensus.cc:348] T d24986c2ebab46f086e3f1f5a53af26b P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:08.466034 19636 raft_consensus.cc:729] T d24986c2ebab46f086e3f1f5a53af26b P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b8d615c71e44ff887e0f037c3efcff7, State: Initialized, Role: LEARNER
I20240502 01:59:08.466125 19636 consensus_queue.cc:260] T d24986c2ebab46f086e3f1f5a53af26b P 3b8d615c71e44ff887e0f037c3efcff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 10, Last appended: 1.10, Last appended by leader: 10, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:08.466380 19636 ts_tablet_manager.cc:1398] T d24986c2ebab46f086e3f1f5a53af26b P 3b8d615c71e44ff887e0f037c3efcff7: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:08.466831 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-d24986c2ebab46f086e3f1f5a53af26b received from {username='slave'} at 127.7.221.72:57673
I20240502 01:59:08.466898 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session 3b8d615c71e44ff887e0f037c3efcff7-d24986c2ebab46f086e3f1f5a53af26b on tablet d24986c2ebab46f086e3f1f5a53af26b with peer 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:08.475282 19636 ts_tablet_manager.cc:902] T 8f2fb504a49c44edac92fd935555c278 P 3b8d615c71e44ff887e0f037c3efcff7: Initiating tablet copy from peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
F20240502 01:59:08.475493 19636 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-1eb15147ff6c4d0b83b12dec736fc576.queue_time_us, tablet-download-pool-1eb15147ff6c4d0b83b12dec736fc576.queue_time_us) (tablet-download-pool-1eb15147ff6c4d0b83b12dec736fc576.run_cpu_time_us, tablet-download-pool-1eb15147ff6c4d0b83b12dec736fc576.run_cpu_time_us) (tablet-download-pool-1eb15147ff6c4d0b83b12dec736fc576.run_wall_time_us, tablet-download-pool-1eb15147ff6c4d0b83b12dec736fc576.run_wall_time_us) (tablet-download-pool-30620f4774b8414eb4e27bd5ecb6ecee.queue_time_us, tablet-download-pool-30620f4774b8414eb4e27bd5ecb6ecee.queue_time_us) (tablet-download-pool-30620f4774b8414eb4e27bd5ecb6ecee.run_cpu_time_us, tablet-download-pool-30620f4774b8414eb4e27bd5ecb6ecee.run_cpu_time_us) (tablet-download-pool-30620f4774b8414eb4e27bd5ecb6ecee.run_wall_time_us, tablet-download-pool-30620f4774b8414eb4e27bd5ecb6ecee.run_wall_time_us) (tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.queue_time_us, tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.queue_time_us) (tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.run_cpu_time_us, tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.run_cpu_time_us) (tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.run_wall_time_us, tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.run_wall_time_us) (tablet-download-pool-5804597cc507483dba622e6ec44e5ead.queue_time_us, tablet-download-pool-5804597cc507483dba622e6ec44e5ead.queue_time_us) (tablet-download-pool-5804597cc507483dba622e6ec44e5ead.run_cpu_time_us, tablet-download-pool-5804597cc507483dba622e6ec44e5ead.run_cpu_time_us) (tablet-download-pool-5804597cc507483dba622e6ec44e5ead.run_wall_time_us, tablet-download-pool-5804597cc507483dba622e6ec44e5ead.run_wall_time_us) (tablet-download-pool-81c8fe21ec7545e79450532b43a61ee7.queue_time_us, tablet-download-pool-81c8fe21ec7545e79450532b43a61ee7.queue_time_us) (tablet-download-pool-81c8fe21ec7545e79450532b43a61ee7.run_cpu_time_us, tablet-download-pool-81c8fe21ec7545e79450532b43a61ee7.run_cpu_time_us) (tablet-download-pool-81c8fe21ec7545e79450532b43a61ee7.run_wall_time_us, tablet-download-pool-81c8fe21ec7545e79450532b43a61ee7.run_wall_time_us) (tablet-download-pool-856f623374824d94913db0cb9faa25d1.queue_time_us, tablet-download-pool-856f623374824d94913db0cb9faa25d1.queue_time_us) (tablet-download-pool-856f623374824d94913db0cb9faa25d1.run_cpu_time_us, tablet-download-pool-856f623374824d94913db0cb9faa25d1.run_cpu_time_us) (tablet-download-pool-856f623374824d94913db0cb9faa25d1.run_wall_time_us, tablet-download-pool-856f623374824d94913db0cb9faa25d1.run_wall_time_us) (tablet-download-pool-8f2fb504a49c44edac92fd935555c278.queue_time_us, tablet-download-pool-8f2fb504a49c44edac92fd935555c278.queue_time_us) (tablet-download-pool-986e7c97e0734e958f364dc5d1167846.queue_time_us, tablet-download-pool-986e7c97e0734e958f364dc5d1167846.queue_time_us) (tablet-download-pool-986e7c97e0734e958f364dc5d1167846.run_cpu_time_us, tablet-download-pool-986e7c97e0734e958f364dc5d1167846.run_cpu_time_us) (tablet-download-pool-986e7c97e0734e958f364dc5d1167846.run_wall_time_us, tablet-download-pool-986e7c97e0734e958f364dc5d1167846.run_wall_time_us) (tablet-download-pool-a901eb3063ea4911a822552db01d365d.queue_time_us, tablet-download-pool-a901eb3063ea4911a822552db01d365d.queue_time_us) (tablet-download-pool-a901eb3063ea4911a822552db01d365d.run_cpu_time_us, tablet-download-pool-a901eb3063ea4911a822552db01d365d.run_cpu_time_us) (tablet-download-pool-a901eb3063ea4911a822552db01d365d.run_wall_time_us, tablet-download-pool-a901eb3063ea4911a822552db01d365d.run_wall_time_us) (tablet-download-pool-ac5ec2c673cb4b988d1099e0442ec63f.queue_time_us, tablet-download-pool-ac5ec2c673cb4b988d1099e0442ec63f.queue_time_us) (tablet-download-pool-ac5ec2c673cb4b988d1099e0442ec63f.run_cpu_time_us, tablet-download-pool-ac5ec2c673cb4b988d1099e0442ec63f.run_cpu_time_us) (tablet-download-pool-ac5ec2c673cb4b988d1099e0442ec63f.run_wall_time_us, tablet-download-pool-ac5ec2c673cb4b988d1099e0442ec63f.run_wall_time_us) (tablet-download-pool-ce85785e670a4264b44797bb27cb4ecb.queue_time_us, tablet-download-pool-ce85785e670a4264b44797bb27cb4ecb.queue_time_us) (tablet-download-pool-ce85785e670a4264b44797bb27cb4ecb.run_cpu_time_us, tablet-download-pool-ce85785e670a4264b44797bb27cb4ecb.run_cpu_time_us) (tablet-download-pool-ce85785e670a4264b44797bb27cb4ecb.run_wall_time_us, tablet-download-pool-ce85785e670a4264b44797bb27cb4ecb.run_wall_time_us) (tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.queue_time_us, tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.queue_time_us) (tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.run_cpu_time_us, tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.run_cpu_time_us) (tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.run_wall_time_us, tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.run_wall_time_us) (tablet-download-pool-e2f4ca025890492880608c7975650969.queue_time_us, tablet-download-pool-e2f4ca025890492880608c7975650969.queue_time_us) (tablet-download-pool-e2f4ca025890492880608c7975650969.run_cpu_time_us, tablet-download-pool-e2f4ca025890492880608c7975650969.run_cpu_time_us) (tablet-download-pool-e2f4ca025890492880608c7975650969.run_wall_time_us, tablet-download-pool-e2f4ca025890492880608c7975650969.run_wall_time_us) (tablet-download-pool-f97c5c9bb4c84957857efc76d1c7717f.queue_time_us, tablet-download-pool-f97c5c9bb4c84957857efc76d1c7717f.queue_time_us) (tablet-download-pool-f97c5c9bb4c84957857efc76d1c7717f.run_cpu_time_us, tablet-download-pool-f97c5c9bb4c84957857efc76d1c7717f.run_cpu_time_us) (tablet-download-pool-f97c5c9bb4c84957857efc76d1c7717f.run_wall_time_us, tablet-download-pool-f97c5c9bb4c84957857efc76d1c7717f.run_wall_time_us) (tablet-download-pool-fc07e430e3e74e3885f263881bd04976.queue_time_us, tablet-download-pool-fc07e430e3e74e3885f263881bd04976.queue_time_us) (tablet-download-pool-fc07e430e3e74e3885f263881bd04976.run_cpu_time_us, tablet-download-pool-fc07e430e3e74e3885f263881bd04976.run_cpu_time_us) (tablet-download-pool-fc07e430e3e74e3885f263881bd04976.run_wall_time_us, tablet-download-pool-fc07e430e3e74e3885f263881bd04976.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
I20240502 01:59:08.476363 19363 ts_tablet_manager.cc:902] T e2f4ca025890492880608c7975650969 P d4e5422f66454511ae5a50ec6ab134ad: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:08.476419 15059 consensus_queue.cc:237] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: true } }
*** Check failure stack trace: ***
I20240502 01:59:08.477936 17242 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
W20240502 01:59:08.478089 14993 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905): Couldn't send request to peer 26a3d5ced97d4dcc919b80fde53f6a5b. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:08.478382 18907 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:08.479357 17104 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:08.479746 18907 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:08.480209 18150 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: true } } }
I20240502 01:59:08.480597 17242 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P 3b8d615c71e44ff887e0f037c3efcff7 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: true } } }
I20240502 01:59:08.483225 14909 catalog_manager.cc:5484] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 9 to 10, NON_VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
@ 0x7f7ccb85fdcd google::LogMessage::Fail() at ??:0
@ 0x7f7ccb863b93 google::LogMessage::SendToLog() at ??:0
@ 0x7f8549d8c7cc google::LogMessage::Flush() at ??:0
@ 0x7f8549d8df59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f7ccb85f7cc google::LogMessage::Flush() at ??:0
@ 0x7f7ccb860f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
I20240502 01:59:08.498715 17104 raft_consensus.cc:2932] T 1eb15147ff6c4d0b83b12dec736fc576 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: true } } }
@ 0x7f7ccccc9044 kudu::TraceMetrics::InternName() at ??:0
I20240502 01:59:08.506994 18880 raft_consensus.cc:2932] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:08.509316 15195 raft_consensus.cc:2932] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:08.510613 17104 raft_consensus.cc:2932] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:08.511421 15059 raft_consensus.cc:2932] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:08.513427 14910 catalog_manager.cc:5484] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 4 to 5, fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
@ 0x7f7ccccadf98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7f854b1f6044 kudu::TraceMetrics::InternName() at ??:0
I20240502 01:59:08.517079 15328 consensus_queue.cc:237] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:08.518457 15195 raft_consensus.cc:1261] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:08.518909 18829 consensus_queue.cc:1035] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:08.519042 17104 raft_consensus.cc:1261] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:08.519518 18829 consensus_queue.cc:1035] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:08.520998 18829 raft_consensus.cc:2932] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:08.522650 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6db29f2516e1486782be0312c9bcac8b with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:08.524910 17104 raft_consensus.cc:2932] T 6db29f2516e1486782be0312c9bcac8b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
@ 0x7f7ccccabefc kudu::ThreadPoolBuilder::Build() at ??:0
I20240502 01:59:08.527244 14915 catalog_manager.cc:5484] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 5 to 6, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:08.527720 15195 raft_consensus.cc:2932] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } }
I20240502 01:59:08.532943 15040 tablet_service.cc:1514] Processing DeleteTablet for tablet 6db29f2516e1486782be0312c9bcac8b with delete_type TABLET_DATA_TOMBSTONED (TS ddae2a636423489693c12b7d7d166c09 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:53384
I20240502 01:59:08.533193 19132 tablet_replica.cc:330] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09: stopping tablet replica
I20240502 01:59:08.533309 19132 raft_consensus.cc:2229] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:08.533458 19132 raft_consensus.cc:2258] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:08.534153 19132 ts_tablet_manager.cc:1875] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:08.536931 19132 ts_tablet_manager.cc:1888] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 01:59:08.536998 19132 log.cc:1196] T 6db29f2516e1486782be0312c9bcac8b P ddae2a636423489693c12b7d7d166c09: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/6db29f2516e1486782be0312c9bcac8b
I20240502 01:59:08.537505 14901 catalog_manager.cc:4830] TS ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123): tablet 6db29f2516e1486782be0312c9bcac8b (table rebalance_test_table_3 [id=0b29d58fe9c74d1cb4345bb6cfbd0535]) successfully deleted
@ 0x7f854b1daf98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7f7ccf85e77a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
I20240502 01:59:08.542714 16973 raft_consensus.cc:1203] T 21e2e55d1a0d4b6eb18b4ade9468607f P d4e5422f66454511ae5a50ec6ab134ad [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
@ 0x7f7ccf867a45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
I20240502 01:59:08.550056 19675 tablet_bootstrap.cc:492] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:08.552088 19675 tablet_bootstrap.cc:492] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:08.552281 19675 ts_tablet_manager.cc:1367] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.136s user 0.024s sys 0.007s
I20240502 01:59:08.552724 19675 raft_consensus.cc:348] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:08.552847 19675 raft_consensus.cc:729] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:08.552942 19675 consensus_queue.cc:260] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
@ 0x7f7ccf8e876f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7f7ccf8fc4f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
I20240502 01:59:08.560222 19675 ts_tablet_manager.cc:1398] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.008s user 0.001s sys 0.000s
I20240502 01:59:08.560686 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-f97c5c9bb4c84957857efc76d1c7717f received from {username='slave'} at 127.7.221.69:43177
I20240502 01:59:08.560763 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-f97c5c9bb4c84957857efc76d1c7717f on tablet f97c5c9bb4c84957857efc76d1c7717f with peer 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:08.561879 15328 consensus_queue.cc:237] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
@ 0x7f7ccf8e7226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
W20240502 01:59:08.563656 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 490e37fac0844a08a07a4b281bc33bf9. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:08.563725 16973 raft_consensus.cc:1261] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:08.564047 15464 raft_consensus.cc:1261] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:08.565408 18417 consensus_queue.cc:1035] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:08.565716 18417 consensus_queue.cc:1035] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:08.568229 18880 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.568584 16973 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.570283 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 490e37fac0844a08a07a4b281bc33bf9 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:08.571170 14909 catalog_manager.cc:5484] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 6 to 7, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:08.571331 15464 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
@ 0x7f854b1d8efc kudu::ThreadPoolBuilder::Build() at ??:0
I20240502 01:59:08.574198 15174 tablet_service.cc:1514] Processing DeleteTablet for tablet 490e37fac0844a08a07a4b281bc33bf9 with delete_type TABLET_DATA_TOMBSTONED (TS 26a3d5ced97d4dcc919b80fde53f6a5b not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:37206
@ 0x7f7ccf8f6c0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55cdd533a75c std::function<>::operator()() at ??:0
@ 0x7f7ccccb0e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f7ccccb16ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f854dd8b77a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7f854dd94a45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
I20240502 01:59:08.596004 17224 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
@ 0x7f7ccccb303b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55cdd533a75c std::function<>::operator()() at ??:0
I20240502 01:59:08.602245 16954 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
@ 0x7f854de1576f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
I20240502 01:59:08.605818 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:08.606200 15174 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
@ 0x7f854de294f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7f7cccca0807 kudu::Thread::SuperviseThread() at ??:0
I20240502 01:59:08.620409 16820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.625563 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:08.627264 15309 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:08.633136 15444 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:08.657332 15328 consensus_queue.cc:237] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true replace: true } }
I20240502 01:59:08.658846 16973 raft_consensus.cc:1261] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:08.658912 15464 raft_consensus.cc:1261] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
@ 0x7f7ccd4de6db start_thread at ??:0
W20240502 01:59:08.659201 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 490e37fac0844a08a07a4b281bc33bf9. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:08.659215 18829 consensus_queue.cc:1035] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:08.659902 18829 consensus_queue.cc:1035] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
@ 0x7f7ccaa1671f clone at ??:0
I20240502 01:59:08.661340 18829 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true replace: true } } }
I20240502 01:59:08.664683 14914 catalog_manager.cc:5484] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 7 to 8. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true replace: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:08.664831 15464 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true replace: true } } }
I20240502 01:59:08.665771 16973 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true replace: true } } }
@ 0x7f854de14226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7f854de23c0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
I20240502 01:59:08.683058 15465 consensus_queue.cc:237] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } }
I20240502 01:59:08.684777 15195 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:08.685150 16973 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:08.685595 19034 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:08.686340 15059 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:08.686798 18766 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:08.687144 18766 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:08.689895 19767 tablet_replica.cc:330] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b: stopping tablet replica
I20240502 01:59:08.690053 19767 raft_consensus.cc:2229] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:08.690258 19767 raft_consensus.cc:2258] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:08.691043 19034 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } }
I20240502 01:59:08.694247 15195 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } }
I20240502 01:59:08.695053 19767 ts_tablet_manager.cc:1875] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:08.695560 14910 catalog_manager.cc:5484] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 7 to 8. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:08.697328 16973 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } }
I20240502 01:59:08.697638 19767 ts_tablet_manager.cc:1888] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 01:59:08.697700 19767 log.cc:1196] T 490e37fac0844a08a07a4b281bc33bf9 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/490e37fac0844a08a07a4b281bc33bf9
I20240502 01:59:08.698527 14900 catalog_manager.cc:4830] TS 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905): tablet 490e37fac0844a08a07a4b281bc33bf9 (table rebalance_test_table_5 [id=3bc69a0a1e474b9fac7330d68770bfbe]) successfully deleted
I20240502 01:59:08.699134 15059 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } }
I20240502 01:59:08.700830 15464 consensus_queue.cc:237] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:08.702440 19831 ts_tablet_manager.cc:902] T 81c8fe21ec7545e79450532b43a61ee7 P f77c2a969bde45b49ac1bbbe242ff171: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
W20240502 01:59:08.703032 15396 consensus_peers.cc:485] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bf73e677bcd049869cadf9f8c9cd3434. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:08.703192 19831 tablet_copy_client.cc:308] T 81c8fe21ec7545e79450532b43a61ee7 P f77c2a969bde45b49ac1bbbe242ff171: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:08.703763 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet 81c8fe21ec7545e79450532b43a61ee7 from peer f77c2a969bde45b49ac1bbbe242ff171 ({username='slave'} at 127.7.221.67:40065)
I20240502 01:59:08.703835 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet 81c8fe21ec7545e79450532b43a61ee7 from peer f77c2a969bde45b49ac1bbbe242ff171 at {username='slave'} at 127.7.221.67:40065: session id = f77c2a969bde45b49ac1bbbe242ff171-81c8fe21ec7545e79450532b43a61ee7
I20240502 01:59:08.704393 15059 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:08.704612 15216 tablet_copy_source_session.cc:205] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:08.704684 15195 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:08.705088 19075 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:08.705545 18296 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:08.705838 19831 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81c8fe21ec7545e79450532b43a61ee7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:08.706144 16973 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:08.706553 18296 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:08.710212 18296 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.710477 19831 tablet_copy_client.cc:783] T 81c8fe21ec7545e79450532b43a61ee7 P f77c2a969bde45b49ac1bbbe242ff171: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:08.710692 19831 tablet_copy_client.cc:647] T 81c8fe21ec7545e79450532b43a61ee7 P f77c2a969bde45b49ac1bbbe242ff171: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:08.712163 14901 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet bf73e677bcd049869cadf9f8c9cd3434 with cas_config_opid_index 8: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 01:59:08.713348 19831 tablet_copy_client.cc:523] T 81c8fe21ec7545e79450532b43a61ee7 P f77c2a969bde45b49ac1bbbe242ff171: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:08.713814 16973 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.714321 14910 catalog_manager.cc:5484] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 8 to 9, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:08.715209 19831 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 01:59:08.715531 15195 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.717003 15059 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.719027 15464 consensus_queue.cc:237] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
W20240502 01:59:08.720976 15396 consensus_peers.cc:485] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bf73e677bcd049869cadf9f8c9cd3434. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:08.721197 15194 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:08.721992 19034 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:08.722769 15059 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:08.723362 19034 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:08.725594 15195 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.770864 19831 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap replayed 1/1 log segments. Stats: ops{read=10 overwritten=0 applied=10 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
@ 0x55f27d75b75c std::function<>::operator()() at ??:0
I20240502 01:59:08.773597 19831 tablet_bootstrap.cc:492] T 81c8fe21ec7545e79450532b43a61ee7 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap complete.
I20240502 01:59:08.773845 19831 ts_tablet_manager.cc:1367] T 81c8fe21ec7545e79450532b43a61ee7 P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.059s user 0.028s sys 0.000s
I20240502 01:59:08.774307 19831 raft_consensus.cc:348] T 81c8fe21ec7545e79450532b43a61ee7 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } }
I20240502 01:59:08.774420 19831 raft_consensus.cc:729] T 81c8fe21ec7545e79450532b43a61ee7 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: LEARNER
I20240502 01:59:08.774509 19831 consensus_queue.cc:260] T 81c8fe21ec7545e79450532b43a61ee7 P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 10, Last appended: 1.10, Last appended by leader: 10, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } }
I20240502 01:59:08.761947 15194 consensus_queue.cc:237] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } }
I20240502 01:59:08.766496 15197 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "856f623374824d94913db0cb9faa25d1"
dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:36028
I20240502 01:59:08.774981 15197 raft_consensus.cc:595] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Received request to transfer leadership
@ 0x7f854b1dde06 kudu::ThreadPool::DispatchThread() at ??:0
I20240502 01:59:08.779220 14910 catalog_manager.cc:5484] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 9 to 10, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.775337 19831 ts_tablet_manager.cc:1398] T 81c8fe21ec7545e79450532b43a61ee7 P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:08.789367 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session f77c2a969bde45b49ac1bbbe242ff171-81c8fe21ec7545e79450532b43a61ee7 received from {username='slave'} at 127.7.221.67:40065
I20240502 01:59:08.789476 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session f77c2a969bde45b49ac1bbbe242ff171-81c8fe21ec7545e79450532b43a61ee7 on tablet 81c8fe21ec7545e79450532b43a61ee7 with peer f77c2a969bde45b49ac1bbbe242ff171
I20240502 01:59:08.776435 15328 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:08.791550 19214 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:08.792593 16954 tablet_service.cc:1514] Processing DeleteTablet for tablet bf73e677bcd049869cadf9f8c9cd3434 with delete_type TABLET_DATA_TOMBSTONED (TS d4e5422f66454511ae5a50ec6ab134ad not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:52144
I20240502 01:59:08.793028 19878 tablet_replica.cc:330] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad: stopping tablet replica
I20240502 01:59:08.793208 19878 raft_consensus.cc:2229] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:08.793430 19878 raft_consensus.cc:2258] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:08.794157 19878 ts_tablet_manager.cc:1875] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:08.796905 19878 ts_tablet_manager.cc:1888] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 01:59:08.797013 19878 log.cc:1196] T bf73e677bcd049869cadf9f8c9cd3434 P d4e5422f66454511ae5a50ec6ab134ad: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal/wals/bf73e677bcd049869cadf9f8c9cd3434
I20240502 01:59:08.797616 14901 catalog_manager.cc:4830] TS d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): tablet bf73e677bcd049869cadf9f8c9cd3434 (table rebalance_test_table_2 [id=73bf3d4779b34c7290b42f7f561e8e11]) successfully deleted
@ 0x7f854b1de6ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
I20240502 01:59:08.724718 19034 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
@ 0x7f854b1e003b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55f27d75b75c std::function<>::operator()() at ??:0
I20240502 01:59:08.827569 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet bf73e677bcd049869cadf9f8c9cd3434 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:08.743180 15464 consensus_queue.cc:237] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 14, Committed index: 14, Last appended: 1.14, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
@ 0x7f854b1cd807 kudu::Thread::SuperviseThread() at ??:0
I20240502 01:59:08.832196 16973 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P d4e5422f66454511ae5a50ec6ab134ad [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 01:59:08.832928 18767 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
W20240502 01:59:08.833937 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
@ 0x7f854ba0b6db start_thread at ??:0
@ 0x7f8548f4371f clone at ??:0
I20240502 01:59:08.835830 18767 raft_consensus.cc:2932] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.15: config changed from index 14 to 15. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:08.726792 15061 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } } }
I20240502 01:59:08.842168 15465 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "3f938c5af51a43c9a23b753c4991e41d"
dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:35070
I20240502 01:59:08.861718 15465 raft_consensus.cc:595] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:59:08.737298 15059 consensus_queue.cc:237] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:08.778328 15057 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:08.864975 18367 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:08.865159 17104 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:08.865945 18907 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:08.847311 19909 ts_tablet_manager.cc:902] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:08.870188 15057 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } }
I20240502 01:59:08.935557 19909 tablet_copy_client.cc:308] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:08.874009 18907 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:08.941327 17104 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:08.866019 14915 catalog_manager.cc:5484] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 14 to 15. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:08.868218 19214 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } }
I20240502 01:59:08.776875 16838 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:08.867094 16836 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:08.884694 16840 raft_consensus.cc:1203] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 01:59:08.938261 14910 catalog_manager.cc:5484] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 8 to 9. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } }
I20240502 01:59:08.885236 15194 consensus_queue.cc:237] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } }
I20240502 01:59:08.936305 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet cd00409d47954ee2b7555e7ff09a6e28 from peer 7b66a8940fe54e21bec1b2db25b2fed5 ({username='slave'} at 127.7.221.68:41001)
I20240502 01:59:08.956774 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet cd00409d47954ee2b7555e7ff09a6e28 from peer 7b66a8940fe54e21bec1b2db25b2fed5 at {username='slave'} at 127.7.221.68:41001: session id = 7b66a8940fe54e21bec1b2db25b2fed5-cd00409d47954ee2b7555e7ff09a6e28
I20240502 01:59:08.945300 19994 ts_tablet_manager.cc:902] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: Initiating tablet copy from peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:08.957691 15216 tablet_copy_source_session.cc:205] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:08.958016 19994 tablet_copy_client.cc:308] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:46123
I20240502 01:59:08.944378 14909 catalog_manager.cc:5484] T e00d1ed5cde044cb9bdebf787590261a P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 9 to 10, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
W20240502 01:59:08.958503 14901 proxy.cc:239] Call had error, refreshing address and retrying: Remote error: Service unavailable: leader transfer in progress
I20240502 01:59:08.960234 19909 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd00409d47954ee2b7555e7ff09a6e28. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:08.960896 15328 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } }
I20240502 01:59:08.963171 15081 tablet_copy_service.cc:140] P ddae2a636423489693c12b7d7d166c09: Received BeginTabletCopySession request for tablet 1eb15147ff6c4d0b83b12dec736fc576 from peer 26a3d5ced97d4dcc919b80fde53f6a5b ({username='slave'} at 127.7.221.66:50987)
W20240502 01:59:08.963218 14901 catalog_manager.cc:4573] TS 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): ChangeConfig:ADD_PEER:NON_VOTER RPC failed for tablet 3f938c5af51a43c9a23b753c4991e41d: Remote error: Service unavailable: leader transfer in progress
I20240502 01:59:08.963274 15081 tablet_copy_service.cc:161] P ddae2a636423489693c12b7d7d166c09: Beginning new tablet copy session on tablet 1eb15147ff6c4d0b83b12dec736fc576 from peer 26a3d5ced97d4dcc919b80fde53f6a5b at {username='slave'} at 127.7.221.66:50987: session id = 26a3d5ced97d4dcc919b80fde53f6a5b-1eb15147ff6c4d0b83b12dec736fc576
I20240502 01:59:08.964159 15081 tablet_copy_source_session.cc:205] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:08.964854 17910 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:08.965440 18319 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:08.966038 19994 ts_tablet_manager.cc:1875] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 01:59:08.967067 16840 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:08.968288 19994 ts_tablet_manager.cc:1888] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.7
I20240502 01:59:08.968451 19994 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1eb15147ff6c4d0b83b12dec736fc576. 1 dirs total, 0 dirs full, 0 dirs failed
W20240502 01:59:08.975718 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): Couldn't send request to peer 7b66a8940fe54e21bec1b2db25b2fed5. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:08.978825 16840 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } }
I20240502 01:59:08.979738 15194 consensus_queue.cc:237] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:08.981181 19994 tablet_copy_client.cc:783] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:08.981369 19994 tablet_copy_client.cc:647] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:08.982196 15329 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:08.984038 18367 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:08.962610 19909 tablet_copy_client.cc:783] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:08.985924 15059 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:08.986052 19909 tablet_copy_client.cc:647] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:08.986367 20027 ts_tablet_manager.cc:902] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5: Initiating tablet copy from peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:08.987079 20027 tablet_copy_client.cc:308] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:46123
I20240502 01:59:08.987521 15081 tablet_copy_service.cc:140] P ddae2a636423489693c12b7d7d166c09: Received BeginTabletCopySession request for tablet 77ae087802144c45924c1b79f561afe8 from peer 7b66a8940fe54e21bec1b2db25b2fed5 ({username='slave'} at 127.7.221.68:58685)
I20240502 01:59:08.987630 15081 tablet_copy_service.cc:161] P ddae2a636423489693c12b7d7d166c09: Beginning new tablet copy session on tablet 77ae087802144c45924c1b79f561afe8 from peer 7b66a8940fe54e21bec1b2db25b2fed5 at {username='slave'} at 127.7.221.68:58685: session id = 7b66a8940fe54e21bec1b2db25b2fed5-77ae087802144c45924c1b79f561afe8
I20240502 01:59:08.987913 18367 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:08.988528 15081 tablet_copy_source_session.cc:205] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:08.988749 19909 tablet_copy_client.cc:523] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:08.989387 20027 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 77ae087802144c45924c1b79f561afe8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:08.990528 19909 tablet_bootstrap.cc:492] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:59:08.991397 20027 tablet_copy_client.cc:783] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:08.991523 20027 tablet_copy_client.cc:647] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:08.991915 18367 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:08.993505 14900 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 6c4e85e7050b4dc8a4ce6bba818854a2 with cas_config_opid_index 9: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 01:59:08.993716 15328 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:08.997017 14910 catalog_manager.cc:5484] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 9 to 10, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:08.998055 15059 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:08.999053 19994 tablet_copy_client.cc:523] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:09.001148 16838 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:09.000794 19994 tablet_bootstrap.cc:492] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
I20240502 01:59:09.001902 18367 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:09.002939 16840 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
W20240502 01:59:09.004921 14995 connection.cc:535] client connection to 127.7.221.72:37645 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 01:59:09.005048 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 01:59:09.005376 15131 connection.cc:535] client connection to 127.7.221.72:37645 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 01:59:09.005478 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 01:59:09.006069 15399 connection.cc:535] client connection to 127.7.221.72:37645 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 01:59:09.006171 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 01:59:09.013352 20027 tablet_copy_client.cc:523] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:09.014732 20027 tablet_bootstrap.cc:492] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
W20240502 01:59:09.015355 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:09.015421 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.020965 15194 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "6c4e85e7050b4dc8a4ce6bba818854a2"
dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:36102
I20240502 01:59:09.021083 15194 raft_consensus.cc:595] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Received request to transfer leadership
W20240502 01:59:09.042958 15266 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:09.049232 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.029868 15194 consensus_queue.cc:237] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } }
W20240502 01:59:09.032133 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.056236 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.056305 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.032344 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.056525 15131 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.056584 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.056677 15131 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.051353 19994 tablet_bootstrap.cc:492] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap replayed 1/1 log segments. Stats: ops{read=10 overwritten=0 applied=10 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
W20240502 01:59:09.057245 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.057561 19994 tablet_bootstrap.cc:492] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap complete.
I20240502 01:59:09.057766 19994 ts_tablet_manager.cc:1367] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.057s user 0.023s sys 0.009s
I20240502 01:59:09.058161 19994 raft_consensus.cc:348] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: true } }
I20240502 01:59:09.058281 19994 raft_consensus.cc:729] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: LEARNER
I20240502 01:59:09.058377 19994 consensus_queue.cc:260] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 10, Last appended: 1.10, Last appended by leader: 10, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: true } }
I20240502 01:59:09.058677 19994 ts_tablet_manager.cc:1398] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
W20240502 01:59:09.060065 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.061079 15328 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:09.061484 15080 tablet_copy_service.cc:342] P ddae2a636423489693c12b7d7d166c09: Request end of tablet copy session 26a3d5ced97d4dcc919b80fde53f6a5b-1eb15147ff6c4d0b83b12dec736fc576 received from {username='slave'} at 127.7.221.66:50987
I20240502 01:59:09.061548 15080 tablet_copy_service.cc:433] P ddae2a636423489693c12b7d7d166c09: ending tablet copy session 26a3d5ced97d4dcc919b80fde53f6a5b-1eb15147ff6c4d0b83b12dec736fc576 on tablet 1eb15147ff6c4d0b83b12dec736fc576 with peer 26a3d5ced97d4dcc919b80fde53f6a5b
I20240502 01:59:09.062256 17471 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:09.061924 15465 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 01:59:09.074908 17910 raft_consensus.cc:979] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b: : Instructing follower 84987ad9e4a54d96af8569e4aaf0ed64 to start an election
I20240502 01:59:09.114269 17910 raft_consensus.cc:1067] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Signalling peer 84987ad9e4a54d96af8569e4aaf0ed64 to start an election
W20240502 01:59:09.062901 15399 consensus_peers.cc:485] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.114655 15399 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.7.221.72:37645, user_credentials={real_user=slave}} blocked reactor thread for 51758.7us
I20240502 01:59:09.063388 19909 tablet_bootstrap.cc:492] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap replayed 1/1 log segments. Stats: ops{read=10 overwritten=0 applied=10 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:09.115461 19909 tablet_bootstrap.cc:492] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap complete.
I20240502 01:59:09.115679 19909 ts_tablet_manager.cc:1367] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.125s user 0.030s sys 0.000s
I20240502 01:59:09.116181 19909 raft_consensus.cc:348] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } }
I20240502 01:59:09.116310 19909 raft_consensus.cc:729] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: LEARNER
I20240502 01:59:09.116417 19909 consensus_queue.cc:260] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 10, Last appended: 1.10, Last appended by leader: 10, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } }
I20240502 01:59:09.116688 19909 ts_tablet_manager.cc:1398] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:09.065073 20027 tablet_bootstrap.cc:492] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:09.117161 16840 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "856f623374824d94913db0cb9faa25d1"
dest_uuid: "84987ad9e4a54d96af8569e4aaf0ed64"
from {username='slave'} at 127.7.221.66:49757
I20240502 01:59:09.117336 20027 tablet_bootstrap.cc:492] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap complete.
I20240502 01:59:09.117336 16840 raft_consensus.cc:482] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 01:59:09.117442 16840 raft_consensus.cc:3037] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:59:09.117503 20027 ts_tablet_manager.cc:1367] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.103s user 0.022s sys 0.004s
I20240502 01:59:09.117923 20027 raft_consensus.cc:348] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } }
I20240502 01:59:09.118023 20027 raft_consensus.cc:729] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: LEARNER
I20240502 01:59:09.118103 20027 consensus_queue.cc:260] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } }
I20240502 01:59:09.118352 20027 ts_tablet_manager.cc:1398] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:09.118620 16840 raft_consensus.cc:504] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:09.119930 16840 leader_election.cc:288] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [CANDIDATE]: Term 2 election: Requested vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207)
I20240502 01:59:09.119956 15080 tablet_copy_service.cc:342] P ddae2a636423489693c12b7d7d166c09: Request end of tablet copy session 7b66a8940fe54e21bec1b2db25b2fed5-77ae087802144c45924c1b79f561afe8 received from {username='slave'} at 127.7.221.68:58685
I20240502 01:59:09.120035 15080 tablet_copy_service.cc:433] P ddae2a636423489693c12b7d7d166c09: ending tablet copy session 7b66a8940fe54e21bec1b2db25b2fed5-77ae087802144c45924c1b79f561afe8 on tablet 77ae087802144c45924c1b79f561afe8 with peer 7b66a8940fe54e21bec1b2db25b2fed5
W20240502 01:59:09.121644 15130 consensus_peers.cc:485] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123): Couldn't send request to peer ddae2a636423489693c12b7d7d166c09. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 21e2e55d1a0d4b6eb18b4ade9468607f. This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.121707 15130 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.7.221.65:46123, user_credentials={real_user=slave}} blocked reactor thread for 58959.3us
I20240502 01:59:09.122826 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session 7b66a8940fe54e21bec1b2db25b2fed5-cd00409d47954ee2b7555e7ff09a6e28 received from {username='slave'} at 127.7.221.68:41001
I20240502 01:59:09.122892 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session 7b66a8940fe54e21bec1b2db25b2fed5-cd00409d47954ee2b7555e7ff09a6e28 on tablet cd00409d47954ee2b7555e7ff09a6e28 with peer 7b66a8940fe54e21bec1b2db25b2fed5
I20240502 01:59:09.122956 19214 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 01:59:09.125258 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "856f623374824d94913db0cb9faa25d1" candidate_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: true dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:59:09.125385 15197 raft_consensus.cc:3032] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Stepping down as leader of term 1
I20240502 01:59:09.125428 15197 raft_consensus.cc:729] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Running, Role: LEADER
I20240502 01:59:09.125588 15197 consensus_queue.cc:260] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:09.125732 15197 raft_consensus.cc:3037] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:59:09.126711 15197 raft_consensus.cc:2454] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84987ad9e4a54d96af8569e4aaf0ed64 in term 2.
I20240502 01:59:09.130757 17104 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "856f623374824d94913db0cb9faa25d1" candidate_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: true dest_uuid: "fc067ee690bb457e8109ad8e4eb43ab8"
I20240502 01:59:09.131729 17104 raft_consensus.cc:3037] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Advancing to term 2
W20240502 01:59:09.132409 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:09.132855 17104 raft_consensus.cc:2454] T 856f623374824d94913db0cb9faa25d1 P fc067ee690bb457e8109ad8e4eb43ab8 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84987ad9e4a54d96af8569e4aaf0ed64 in term 2.
I20240502 01:59:09.133190 16771 leader_election.cc:302] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 84987ad9e4a54d96af8569e4aaf0ed64, fc067ee690bb457e8109ad8e4eb43ab8; no voters:
I20240502 01:59:09.133406 19703 raft_consensus.cc:2781] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 01:59:09.133546 19703 raft_consensus.cc:686] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 2 LEADER]: Becoming Leader. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Running, Role: LEADER
W20240502 01:59:09.133546 16774 leader_election.cc:334] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:09.133828 19703 consensus_queue.cc:237] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } }
I20240502 01:59:09.139487 17910 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } } }
I20240502 01:59:09.141736 14906 catalog_manager.cc:5484] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 reported cstate change: term changed from 1 to 2, leader changed from 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) to 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69). New cstate: current_term: 2 leader_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:09.144243 15465 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } } }
I20240502 01:59:09.145151 14910 catalog_manager.cc:5484] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 4 to 5, NON_VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:09.157202 15174 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
W20240502 01:59:09.162511 15265 connection.cc:535] client connection to 127.7.221.70:34649 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 01:59:09.162969 15130 connection.cc:535] client connection to 127.7.221.70:34649 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 01:59:09.164227 14994 connection.cc:535] client connection to 127.7.221.70:34649 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 01:59:09.165401 20044 negotiation.cc:337] Failed RPC negotiation. Trace:
0502 01:59:09.163755 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.7.221.70:34649 (local address 127.7.221.68:40101)
0502 01:59:09.163900 (+ 145us) negotiation.cc:107] Waiting for socket to connect
0502 01:59:09.163913 (+ 13us) client_negotiation.cc:174] Beginning negotiation
0502 01:59:09.164002 (+ 89us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
0502 01:59:09.164831 (+ 829us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
Metrics: {"client-negotiator.queue_time_us":66}
W20240502 01:59:09.165408 20038 negotiation.cc:337] Failed RPC negotiation. Trace:
0502 01:59:09.164792 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.7.221.70:34649 (local address 127.7.221.65:60995)
0502 01:59:09.165069 (+ 277us) negotiation.cc:107] Waiting for socket to connect
0502 01:59:09.165112 (+ 43us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection reset by peer (error 104)
Metrics: {"client-negotiator.queue_time_us":211}
W20240502 01:59:09.165792 20051 negotiation.cc:337] Failed RPC negotiation. Trace:
0502 01:59:09.163726 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.7.221.70:34649 (local address 127.7.221.67:54409)
0502 01:59:09.163909 (+ 183us) negotiation.cc:107] Waiting for socket to connect
0502 01:59:09.163921 (+ 12us) client_negotiation.cc:174] Beginning negotiation
0502 01:59:09.164006 (+ 85us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
0502 01:59:09.165330 (+ 1324us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
Metrics: {"client-negotiator.queue_time_us":109}
W20240502 01:59:09.165788 14994 consensus_peers.cc:462] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection reset by peer (error 104)
W20240502 01:59:09.165861 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection reset by peer (error 104). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.165928 15265 consensus_peers.cc:485] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.166006 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.166007 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.166060 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 01:59:09.167832 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.167912 15130 consensus_peers.cc:485] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.167945 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.172446 15309 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:09.172986 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:09.175631 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:09.177290 16820 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:59:09.180549 15444 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
W20240502 01:59:09.183995 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.192122 15328 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } } }
I20240502 01:59:09.203649 15328 raft_consensus.cc:1203] T 81c8fe21ec7545e79450532b43a61ee7 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.9->[1.10-1.10] Dedup: 1.10->[]
I20240502 01:59:09.207175 15059 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "8ddc239a297a46e48e069b985cb7b350"
dest_uuid: "ddae2a636423489693c12b7d7d166c09"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:57914
I20240502 01:59:09.207374 15059 raft_consensus.cc:595] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:59:09.220885 15197 consensus_queue.cc:237] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
W20240502 01:59:09.223851 15130 consensus_peers.cc:485] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123): Couldn't send request to peer ddae2a636423489693c12b7d7d166c09. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 21e2e55d1a0d4b6eb18b4ade9468607f. This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.224143 15130 consensus_peers.cc:485] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.224150 15328 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:09.225598 18109 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:09.227933 19214 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:09.228483 15328 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
W20240502 01:59:09.230265 15131 consensus_peers.cc:485] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.230759 15465 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 01:59:09.230913 14910 catalog_manager.cc:5484] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 5 to 6, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:09.233355 17471 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 01:59:09.235285 15465 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:09.236204 20165 ts_tablet_manager.cc:902] T 490e37fac0844a08a07a4b281bc33bf9 P fc067ee690bb457e8109ad8e4eb43ab8: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:09.236780 20165 tablet_copy_client.cc:308] T 490e37fac0844a08a07a4b281bc33bf9 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 01:59:09.238667 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet 490e37fac0844a08a07a4b281bc33bf9 from peer fc067ee690bb457e8109ad8e4eb43ab8 ({username='slave'} at 127.7.221.71:41349)
I20240502 01:59:09.238781 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet 490e37fac0844a08a07a4b281bc33bf9 from peer fc067ee690bb457e8109ad8e4eb43ab8 at {username='slave'} at 127.7.221.71:41349: session id = fc067ee690bb457e8109ad8e4eb43ab8-490e37fac0844a08a07a4b281bc33bf9
I20240502 01:59:09.239759 15351 tablet_copy_source_session.cc:205] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:09.241569 20165 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 490e37fac0844a08a07a4b281bc33bf9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:09.251724 15197 consensus_queue.cc:237] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } }
I20240502 01:59:09.253432 17104 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:09.253783 17471 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
W20240502 01:59:09.254169 15130 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): Couldn't send request to peer 7b66a8940fe54e21bec1b2db25b2fed5. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.254173 16840 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:09.254747 17471 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
W20240502 01:59:09.255527 15131 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.260396 20165 tablet_copy_client.cc:783] T 490e37fac0844a08a07a4b281bc33bf9 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:09.260617 20165 tablet_copy_client.cc:647] T 490e37fac0844a08a07a4b281bc33bf9 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:09.261873 17471 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } }
W20240502 01:59:09.262756 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:09.264513 16840 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } }
I20240502 01:59:09.265765 17104 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } }
I20240502 01:59:09.267333 14912 catalog_manager.cc:5484] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 10 to 11, NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:09.278134 15328 consensus_queue.cc:237] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
W20240502 01:59:09.280539 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.281422 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.281719 17104 raft_consensus.cc:1261] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:09.282130 18416 consensus_queue.cc:1035] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:09.284487 20213 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:09.285638 17296 rebalancer_tool-test.cc:1260] rebalancer run info: Runtime error: /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu: process exited with non-zero status 1
stdout: Per-server replica distribution summary:
Statistic | Value
-----------------------+-----------
Minimum Replica Count | 0
Maximum Replica Count | 23
Average Replica Count | 11.250000
Per-table replica distribution summary:
Replica Skew | Value
--------------+----------
Minimum | 3
Maximum | 3
Average | 3.000000
stderr: W20240502 01:59:03.205140 17305 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:59:03.205423 17305 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:59:03.209277 17305 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 01:59:04.205931 17305 rebalancer_tool.cc:215] running whole-cluster rebalancing
W20240502 01:59:04.672849 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.672961 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.673046 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 9736e7c5365849759564e386d208932f: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:04.673115 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.673177 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.673228 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.673286 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.673332 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:04.673386 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.673427 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0b29d58fe9c74d1cb4345bb6cfbd0535: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.673480 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.673519 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.673557 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.673609 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:04.705514 17305 rebalancer_tool.cc:1074] tablet e2f4ca025890492880608c7975650969: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:04.740796 17305 rebalancer_tool.cc:1074] tablet 81c8fe21ec7545e79450532b43a61ee7: 'ddae2a636423489693c12b7d7d166c09' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:04.785066 17305 rebalancer_tool.cc:1074] tablet d1c832c721fb4b15bc6d266a1c295401: 'f77c2a969bde45b49ac1bbbe242ff171' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:04.839113 17305 rebalancer_tool.cc:1074] tablet 986e7c97e0734e958f364dc5d1167846: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move scheduled
I20240502 01:59:04.926942 17305 rebalancer_tool.cc:1074] tablet 9c2892024ce942e1bf1834c1b9ad2049: '7b66a8940fe54e21bec1b2db25b2fed5' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:05.034545 17305 rebalancer_tool.cc:1074] tablet fc07e430e3e74e3885f263881bd04976: 'ddae2a636423489693c12b7d7d166c09' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:05.103387 17305 rebalancer_tool.cc:1074] tablet ac5ec2c673cb4b988d1099e0442ec63f: 'f77c2a969bde45b49ac1bbbe242ff171' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:05.154215 17305 rebalancer_tool.cc:1074] tablet 1eb15147ff6c4d0b83b12dec736fc576: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move scheduled
I20240502 01:59:05.179894 17305 rebalancer_tool.cc:1084] tablet 8f2fb504a49c44edac92fd935555c278: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'd4e5422f66454511ae5a50ec6ab134ad' move ignored: Illegal state: tablet 8f2fb504a49c44edac92fd935555c278: replica d4e5422f66454511ae5a50ec6ab134ad is already present
I20240502 01:59:05.179996 17305 rebalancer_tool.cc:756] re-synchronizing cluster state
W20240502 01:59:06.170048 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:06.234645 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:06.234799 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:06.234853 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server 84987ad9e4a54d96af8569e4aaf0ed64 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:06.234884 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0b29d58fe9c74d1cb4345bb6cfbd0535: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:06.234910 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:06.234939 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:06.234962 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:06.234999 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 9736e7c5365849759564e386d208932f: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:06.235023 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:06.235056 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:06.235085 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:06.235122 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:06.235160 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:06.235214 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0b29d58fe9c74d1cb4345bb6cfbd0535: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:06.235251 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:06.340842 17305 rebalancer_tool.cc:1074] tablet 3f938c5af51a43c9a23b753c4991e41d: '7b66a8940fe54e21bec1b2db25b2fed5' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:06.412674 17305 rebalancer_tool.cc:1084] tablet cd00409d47954ee2b7555e7ff09a6e28: 'ddae2a636423489693c12b7d7d166c09' -> 'd4e5422f66454511ae5a50ec6ab134ad' move ignored: Illegal state: tablet cd00409d47954ee2b7555e7ff09a6e28: replica d4e5422f66454511ae5a50ec6ab134ad is already present
I20240502 01:59:06.412757 17305 rebalancer_tool.cc:756] re-synchronizing cluster state
W20240502 01:59:07.613751 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:07.613826 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server 3b8d615c71e44ff887e0f037c3efcff7 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:07.613896 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:07.613947 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:07.613987 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 9736e7c5365849759564e386d208932f: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:07.614027 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 7b66a8940fe54e21bec1b2db25b2fed5
W20240502 01:59:07.614061 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server ddae2a636423489693c12b7d7d166c09
W20240502 01:59:07.614104 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:07.614161 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:07.614197 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:07.614238 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:07.614272 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:07.673111 17305 rebalancer_tool.cc:1074] tablet 21e2e55d1a0d4b6eb18b4ade9468607f: 'f77c2a969bde45b49ac1bbbe242ff171' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:07.736428 17305 rebalancer_tool.cc:1074] tablet d24986c2ebab46f086e3f1f5a53af26b: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:07.896479 17305 rebalancer_tool.cc:1084] tablet 5804597cc507483dba622e6ec44e5ead: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'd4e5422f66454511ae5a50ec6ab134ad' move ignored: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }.
Pending config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:07.896559 17305 rebalancer_tool.cc:756] re-synchronizing cluster state
W20240502 01:59:08.460690 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 9736e7c5365849759564e386d208932f: could not find any suitable replica to move from server d4e5422f66454511ae5a50ec6ab134ad to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:08.460855 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server 84987ad9e4a54d96af8569e4aaf0ed64 to server 7b66a8940fe54e21bec1b2db25b2fed5
W20240502 01:59:08.461040 17305 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:08.667804 17305 rebalancer_tool.cc:1074] tablet 490e37fac0844a08a07a4b281bc33bf9: 'fc067ee690bb457e8109ad8e4eb43ab8' -> '' move scheduled
I20240502 01:59:08.693729 17305 rebalancer_tool.cc:1074] tablet bf73e677bcd049869cadf9f8c9cd3434: 'd4e5422f66454511ae5a50ec6ab134ad' -> '' move scheduled
I20240502 01:59:08.939383 17305 rebalancer_tool.cc:1074] tablet e00d1ed5cde044cb9bdebf787590261a: 'ddae2a636423489693c12b7d7d166c09' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:08.976270 17305 rebalancer_tool.cc:1084] tablet 77ae087802144c45924c1b79f561afe8: '84987ad9e4a54d96af8569e4aaf0ed64' -> '7b66a8940fe54e21bec1b2db25b2fed5' move ignored: Illegal state: tablet 77ae087802144c45924c1b79f561afe8: replica 7b66a8940fe54e21bec1b2db25b2fed5 is already present
I20240502 01:59:08.976379 17305 rebalancer_tool.cc:756] re-synchronizing cluster state
W20240502 01:59:09.226434 20134 negotiation.cc:337] Failed RPC negotiation. Trace:
0502 01:59:09.154276 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.7.221.70:34649 (local address 127.0.0.1:52234)
0502 01:59:09.156126 (+ 1850us) negotiation.cc:107] Waiting for socket to connect
0502 01:59:09.156137 (+ 11us) client_negotiation.cc:174] Beginning negotiation
0502 01:59:09.156223 (+ 86us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
0502 01:59:09.175146 (+ 18923us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
Metrics: {"client-negotiator.queue_time_us":1585,"thread_start_us":45,"threads_started":1}
Illegal state: tablet server 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): unacceptable health status UNAVAILABLE
I20240502 01:59:09.289791 17104 raft_consensus.cc:2932] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:09.291150 20165 tablet_copy_client.cc:523] T 490e37fac0844a08a07a4b281bc33bf9 P fc067ee690bb457e8109ad8e4eb43ab8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:09.292702 14906 catalog_manager.cc:5484] T ce85785e670a4264b44797bb27cb4ecb P fc067ee690bb457e8109ad8e4eb43ab8 reported cstate change: config changed from index 9 to 10, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:09.296144 20165 tablet_bootstrap.cc:492] T 490e37fac0844a08a07a4b281bc33bf9 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap starting.
I20240502 01:59:09.313177 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:09.314249 15444 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:09.316781 15174 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 01:59:09.338335 15465 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846"
dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:35140
I20240502 01:59:09.338475 15465 raft_consensus.cc:595] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:59:09.345523 20165 tablet_bootstrap.cc:492] T 490e37fac0844a08a07a4b281bc33bf9 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:09.345999 20165 tablet_bootstrap.cc:492] T 490e37fac0844a08a07a4b281bc33bf9 P fc067ee690bb457e8109ad8e4eb43ab8: Bootstrap complete.
I20240502 01:59:09.346257 20165 ts_tablet_manager.cc:1367] T 490e37fac0844a08a07a4b281bc33bf9 P fc067ee690bb457e8109ad8e4eb43ab8: Time spent bootstrapping tablet: real 0.050s user 0.028s sys 0.000s
I20240502 01:59:09.346419 16820 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 01:59:09.346486 17087 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 01:59:09.346765 15309 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:09.346993 20165 raft_consensus.cc:348] T 490e37fac0844a08a07a4b281bc33bf9 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true replace: true } }
I20240502 01:59:09.347173 20165 raft_consensus.cc:729] T 490e37fac0844a08a07a4b281bc33bf9 P fc067ee690bb457e8109ad8e4eb43ab8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fc067ee690bb457e8109ad8e4eb43ab8, State: Initialized, Role: LEARNER
I20240502 01:59:09.347338 20165 consensus_queue.cc:260] T 490e37fac0844a08a07a4b281bc33bf9 P fc067ee690bb457e8109ad8e4eb43ab8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true replace: true } }
I20240502 01:59:09.347802 20165 ts_tablet_manager.cc:1398] T 490e37fac0844a08a07a4b281bc33bf9 P fc067ee690bb457e8109ad8e4eb43ab8: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:09.349133 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-490e37fac0844a08a07a4b281bc33bf9 received from {username='slave'} at 127.7.221.71:41349
I20240502 01:59:09.349246 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session fc067ee690bb457e8109ad8e4eb43ab8-490e37fac0844a08a07a4b281bc33bf9 on tablet 490e37fac0844a08a07a4b281bc33bf9 with peer fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:09.354831 15465 raft_consensus.cc:1203] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.9->[1.10-1.10] Dedup: 1.10->[]
W20240502 01:59:09.359798 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 01:59:09.362852 20165 ts_tablet_manager.cc:902] T bf73e677bcd049869cadf9f8c9cd3434 P fc067ee690bb457e8109ad8e4eb43ab8: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
F20240502 01:59:09.363106 20165 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-1eb15147ff6c4d0b83b12dec736fc576.queue_time_us, tablet-download-pool-1eb15147ff6c4d0b83b12dec736fc576.queue_time_us) (tablet-download-pool-1eb15147ff6c4d0b83b12dec736fc576.run_cpu_time_us, tablet-download-pool-1eb15147ff6c4d0b83b12dec736fc576.run_cpu_time_us) (tablet-download-pool-1eb15147ff6c4d0b83b12dec736fc576.run_wall_time_us, tablet-download-pool-1eb15147ff6c4d0b83b12dec736fc576.run_wall_time_us) (tablet-download-pool-490e37fac0844a08a07a4b281bc33bf9.queue_time_us, tablet-download-pool-490e37fac0844a08a07a4b281bc33bf9.queue_time_us) (tablet-download-pool-490e37fac0844a08a07a4b281bc33bf9.run_cpu_time_us, tablet-download-pool-490e37fac0844a08a07a4b281bc33bf9.run_cpu_time_us) (tablet-download-pool-490e37fac0844a08a07a4b281bc33bf9.run_wall_time_us, tablet-download-pool-490e37fac0844a08a07a4b281bc33bf9.run_wall_time_us) (tablet-download-pool-53edbd87d0d5471ca7626ca2071f75eb.queue_time_us, tablet-download-pool-53edbd87d0d5471ca7626ca2071f75eb.queue_time_us) (tablet-download-pool-53edbd87d0d5471ca7626ca2071f75eb.run_cpu_time_us, tablet-download-pool-53edbd87d0d5471ca7626ca2071f75eb.run_cpu_time_us) (tablet-download-pool-53edbd87d0d5471ca7626ca2071f75eb.run_wall_time_us, tablet-download-pool-53edbd87d0d5471ca7626ca2071f75eb.run_wall_time_us) (tablet-download-pool-6c4e85e7050b4dc8a4ce6bba818854a2.queue_time_us, tablet-download-pool-6c4e85e7050b4dc8a4ce6bba818854a2.queue_time_us) (tablet-download-pool-6c4e85e7050b4dc8a4ce6bba818854a2.run_cpu_time_us, tablet-download-pool-6c4e85e7050b4dc8a4ce6bba818854a2.run_cpu_time_us) (tablet-download-pool-6c4e85e7050b4dc8a4ce6bba818854a2.run_wall_time_us, tablet-download-pool-6c4e85e7050b4dc8a4ce6bba818854a2.run_wall_time_us) (tablet-download-pool-6d5cffb7fcc54118861463e3b1fa4a8d.queue_time_us, tablet-download-pool-6d5cffb7fcc54118861463e3b1fa4a8d.queue_time_us) (tablet-download-pool-6d5cffb7fcc54118861463e3b1fa4a8d.run_cpu_time_us, tablet-download-pool-6d5cffb7fcc54118861463e3b1fa4a8d.run_cpu_time_us) (tablet-download-pool-6d5cffb7fcc54118861463e3b1fa4a8d.run_wall_time_us, tablet-download-pool-6d5cffb7fcc54118861463e3b1fa4a8d.run_wall_time_us) (tablet-download-pool-6db29f2516e1486782be0312c9bcac8b.queue_time_us, tablet-download-pool-6db29f2516e1486782be0312c9bcac8b.queue_time_us) (tablet-download-pool-6db29f2516e1486782be0312c9bcac8b.run_cpu_time_us, tablet-download-pool-6db29f2516e1486782be0312c9bcac8b.run_cpu_time_us) (tablet-download-pool-6db29f2516e1486782be0312c9bcac8b.run_wall_time_us, tablet-download-pool-6db29f2516e1486782be0312c9bcac8b.run_wall_time_us) (tablet-download-pool-856f623374824d94913db0cb9faa25d1.queue_time_us, tablet-download-pool-856f623374824d94913db0cb9faa25d1.queue_time_us) (tablet-download-pool-856f623374824d94913db0cb9faa25d1.run_cpu_time_us, tablet-download-pool-856f623374824d94913db0cb9faa25d1.run_cpu_time_us) (tablet-download-pool-856f623374824d94913db0cb9faa25d1.run_wall_time_us, tablet-download-pool-856f623374824d94913db0cb9faa25d1.run_wall_time_us) (tablet-download-pool-986e7c97e0734e958f364dc5d1167846.queue_time_us, tablet-download-pool-986e7c97e0734e958f364dc5d1167846.queue_time_us) (tablet-download-pool-986e7c97e0734e958f364dc5d1167846.run_cpu_time_us, tablet-download-pool-986e7c97e0734e958f364dc5d1167846.run_cpu_time_us) (tablet-download-pool-986e7c97e0734e958f364dc5d1167846.run_wall_time_us, tablet-download-pool-986e7c97e0734e958f364dc5d1167846.run_wall_time_us) (tablet-download-pool-ac5ec2c673cb4b988d1099e0442ec63f.queue_time_us, tablet-download-pool-ac5ec2c673cb4b988d1099e0442ec63f.queue_time_us) (tablet-download-pool-ac5ec2c673cb4b988d1099e0442ec63f.run_cpu_time_us, tablet-download-pool-ac5ec2c673cb4b988d1099e0442ec63f.run_cpu_time_us) (tablet-download-pool-ac5ec2c673cb4b988d1099e0442ec63f.run_wall_time_us, tablet-download-pool-ac5ec2c673cb4b988d1099e0442ec63f.run_wall_time_us) (tablet-download-pool-bf73e677bcd049869cadf9f8c9cd3434.queue_time_us, tablet-download-pool-bf73e677bcd049869cadf9f8c9cd3434.queue_time_us) (tablet-download-pool-cd00409d47954ee2b7555e7ff09a6e28.queue_time_us, tablet-download-pool-cd00409d47954ee2b7555e7ff09a6e28.queue_time_us) (tablet-download-pool-cd00409d47954ee2b7555e7ff09a6e28.run_cpu_time_us, tablet-download-pool-cd00409d47954ee2b7555e7ff09a6e28.run_cpu_time_us) (tablet-download-pool-cd00409d47954ee2b7555e7ff09a6e28.run_wall_time_us, tablet-download-pool-cd00409d47954ee2b7555e7ff09a6e28.run_wall_time_us) (tablet-download-pool-ce85785e670a4264b44797bb27cb4ecb.queue_time_us, tablet-download-pool-ce85785e670a4264b44797bb27cb4ecb.queue_time_us) (tablet-download-pool-ce85785e670a4264b44797bb27cb4ecb.run_cpu_time_us, tablet-download-pool-ce85785e670a4264b44797bb27cb4ecb.run_cpu_time_us) (tablet-download-pool-ce85785e670a4264b44797bb27cb4ecb.run_wall_time_us, tablet-download-pool-ce85785e670a4264b44797bb27cb4ecb.run_wall_time_us) (tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.queue_time_us, tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.queue_time_us) (tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.run_cpu_time_us, tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.run_cpu_time_us) (tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.run_wall_time_us, tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.run_wall_time_us) (tablet-download-pool-e00d1ed5cde044cb9bdebf787590261a.queue_time_us, tablet-download-pool-e00d1ed5cde044cb9bdebf787590261a.queue_time_us) (tablet-download-pool-e00d1ed5cde044cb9bdebf787590261a.run_cpu_time_us, tablet-download-pool-e00d1ed5cde044cb9bdebf787590261a.run_cpu_time_us) (tablet-download-pool-e00d1ed5cde044cb9bdebf787590261a.run_wall_time_us, tablet-download-pool-e00d1ed5cde044cb9bdebf787590261a.run_wall_time_us) (tablet-download-pool-fcfc8dc4b90a4a6aa7e4035bbfdb4532.queue_time_us, tablet-download-pool-fcfc8dc4b90a4a6aa7e4035bbfdb4532.queue_time_us) (tablet-download-pool-fcfc8dc4b90a4a6aa7e4035bbfdb4532.run_cpu_time_us, tablet-download-pool-fcfc8dc4b90a4a6aa7e4035bbfdb4532.run_cpu_time_us) (tablet-download-pool-fcfc8dc4b90a4a6aa7e4035bbfdb4532.run_wall_time_us, tablet-download-pool-fcfc8dc4b90a4a6aa7e4035bbfdb4532.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7f6b26a33dcd google::LogMessage::Fail() at ??:0
W20240502 01:59:09.368331 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
@ 0x7f6b26a37b93 google::LogMessage::SendToLog() at ??:0
@ 0x7f6b26a337cc google::LogMessage::Flush() at ??:0
@ 0x7f6b26a34f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
W20240502 01:59:09.377400 14994 consensus_peers.cc:462] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
@ 0x7f6b27e9d044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7f6b27e81f98 kudu::ThreadPool::ThreadPool() at ??:0
I20240502 01:59:09.382817 19214 raft_consensus.cc:979] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b: : Instructing follower ddae2a636423489693c12b7d7d166c09 to start an election
I20240502 01:59:09.382910 19214 raft_consensus.cc:1067] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Signalling peer ddae2a636423489693c12b7d7d166c09 to start an election
@ 0x7f6b27e7fefc kudu::ThreadPoolBuilder::Build() at ??:0
I20240502 01:59:09.384653 15059 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "6c4e85e7050b4dc8a4ce6bba818854a2"
dest_uuid: "ddae2a636423489693c12b7d7d166c09"
from {username='slave'} at 127.7.221.66:50987
I20240502 01:59:09.384815 15059 raft_consensus.cc:482] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 01:59:09.384877 15059 raft_consensus.cc:3037] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:59:09.386139 15059 raft_consensus.cc:504] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:09.386801 15059 leader_election.cc:288] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 2 election: Requested vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223)
I20240502 01:59:09.387506 16840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6c4e85e7050b4dc8a4ce6bba818854a2" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 2 candidate_status { last_received { term: 1 index: 10 } } ignore_live_leader: true dest_uuid: "84987ad9e4a54d96af8569e4aaf0ed64"
I20240502 01:59:09.387701 16840 raft_consensus.cc:3037] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:59:09.387737 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6c4e85e7050b4dc8a4ce6bba818854a2" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 2 candidate_status { last_received { term: 1 index: 10 } } ignore_live_leader: true dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:59:09.387830 15328 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6c4e85e7050b4dc8a4ce6bba818854a2" candidate_uuid: "ddae2a636423489693c12b7d7d166c09" candidate_term: 2 candidate_status { last_received { term: 1 index: 10 } } ignore_live_leader: true dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
I20240502 01:59:09.387871 15197 raft_consensus.cc:3032] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Stepping down as leader of term 1
I20240502 01:59:09.387913 15197 raft_consensus.cc:729] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Running, Role: LEADER
I20240502 01:59:09.387964 15328 raft_consensus.cc:3037] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:59:09.388077 15197 consensus_queue.cc:260] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 10, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:09.388326 15197 raft_consensus.cc:3037] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:59:09.388814 16840 raft_consensus.cc:2454] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 2.
I20240502 01:59:09.388988 15328 raft_consensus.cc:2454] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 2.
I20240502 01:59:09.389472 15197 raft_consensus.cc:2454] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ddae2a636423489693c12b7d7d166c09 in term 2.
I20240502 01:59:09.389478 14993 leader_election.cc:302] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 84987ad9e4a54d96af8569e4aaf0ed64, ddae2a636423489693c12b7d7d166c09, f77c2a969bde45b49ac1bbbe242ff171; no voters:
@ 0x7f6b2aa3277a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
I20240502 01:59:09.389695 18319 raft_consensus.cc:2781] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 01:59:09.389793 18319 raft_consensus.cc:686] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 2 LEADER]: Becoming Leader. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Running, Role: LEADER
I20240502 01:59:09.389989 18319 consensus_queue.cc:237] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 10, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
@ 0x7f6b2aa3ba45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
I20240502 01:59:09.396831 14909 catalog_manager.cc:5484] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 reported cstate change: term changed from 1 to 2, leader changed from 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) to ddae2a636423489693c12b7d7d166c09 (127.7.221.65). New cstate: current_term: 2 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
@ 0x7f6b2aabc76f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
I20240502 01:59:09.400202 15465 raft_consensus.cc:1203] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
@ 0x7f6b2aad04f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7f6b2aabb226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7f6b2aacac0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x561f590fd75c std::function<>::operator()() at ??:0
@ 0x7f6b27e84e06 kudu::ThreadPool::DispatchThread() at ??:0
I20240502 01:59:09.417765 17298 rebalancer_tool-test.cc:1260] rebalancer run info: Runtime error: /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu: process exited with non-zero status 1
stdout: Per-server replica distribution summary:
Statistic | Value
-----------------------+-----------
Minimum Replica Count | 0
Maximum Replica Count | 23
Average Replica Count | 11.250000
Per-table replica distribution summary:
Replica Skew | Value
--------------+----------
Minimum | 3
Maximum | 3
Average | 3.000000
stderr: W20240502 01:59:03.167981 17304 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:59:03.168319 17304 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:59:03.172821 17304 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 01:59:04.199959 17304 rebalancer_tool.cc:215] running whole-cluster rebalancing
W20240502 01:59:04.721755 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0b29d58fe9c74d1cb4345bb6cfbd0535: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.721876 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.721930 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.721978 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 9736e7c5365849759564e386d208932f: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.722009 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.722075 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:04.722103 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.722133 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.722162 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.722194 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:04.722247 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.722293 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:04.722327 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.722354 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:04.757431 17304 rebalancer_tool.cc:1074] tablet e2f4ca025890492880608c7975650969: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:04.804533 17304 rebalancer_tool.cc:1074] tablet a901eb3063ea4911a822552db01d365d: 'f77c2a969bde45b49ac1bbbe242ff171' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:04.863381 17304 rebalancer_tool.cc:1074] tablet d24986c2ebab46f086e3f1f5a53af26b: 'ddae2a636423489693c12b7d7d166c09' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:04.956019 17304 rebalancer_tool.cc:1074] tablet ac5ec2c673cb4b988d1099e0442ec63f: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move scheduled
I20240502 01:59:05.037802 17304 rebalancer_tool.cc:1074] tablet 8f2fb504a49c44edac92fd935555c278: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:05.078858 17304 rebalancer_tool.cc:1074] tablet 5804597cc507483dba622e6ec44e5ead: 'f77c2a969bde45b49ac1bbbe242ff171' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:05.106359 17304 rebalancer_tool.cc:1074] tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532: 'ddae2a636423489693c12b7d7d166c09' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:05.134476 17304 rebalancer_tool.cc:1074] tablet 856f623374824d94913db0cb9faa25d1: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move scheduled
I20240502 01:59:05.168233 17304 rebalancer_tool.cc:1074] tablet 30620f4774b8414eb4e27bd5ecb6ecee: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:05.305377 17304 rebalancer_tool.cc:1074] tablet d1c832c721fb4b15bc6d266a1c295401: 'f77c2a969bde45b49ac1bbbe242ff171' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:05.417743 17304 rebalancer_tool.cc:1074] tablet 6d5cffb7fcc54118861463e3b1fa4a8d: 'ddae2a636423489693c12b7d7d166c09' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:05.600845 17304 rebalancer_tool.cc:1074] tablet 53edbd87d0d5471ca7626ca2071f75eb: 'f77c2a969bde45b49ac1bbbe242ff171' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move scheduled
I20240502 01:59:05.683576 17304 rebalancer_tool.cc:1074] tablet 091d27d51acc4db0a1c7d20deeff5c10: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:05.823568 17304 rebalancer_tool.cc:1074] tablet 6c4e85e7050b4dc8a4ce6bba818854a2: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:06.001782 17304 rebalancer_tool.cc:1074] tablet ce85785e670a4264b44797bb27cb4ecb: 'ddae2a636423489693c12b7d7d166c09' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:06.225951 17304 rebalancer_tool.cc:1084] tablet e00d1ed5cde044cb9bdebf787590261a: 'f77c2a969bde45b49ac1bbbe242ff171' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move ignored: Illegal state: tablet e00d1ed5cde044cb9bdebf787590261a: replica fc067ee690bb457e8109ad8e4eb43ab8 is already present
I20240502 01:59:06.226052 17304 rebalancer_tool.cc:756] re-synchronizing cluster state
W20240502 01:59:07.166925 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:07.167085 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:07.167217 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:07.167277 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:07.168838 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:07.485643 17304 rebalancer_tool.cc:1074] tablet bf73e677bcd049869cadf9f8c9cd3434: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move scheduled
I20240502 01:59:07.656193 17304 rebalancer_tool.cc:1074] tablet f97c5c9bb4c84957857efc76d1c7717f: '3b8d615c71e44ff887e0f037c3efcff7' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:07.723238 17304 rebalancer_tool.cc:1084] tablet e00d1ed5cde044cb9bdebf787590261a: 'f77c2a969bde45b49ac1bbbe242ff171' -> 'd4e5422f66454511ae5a50ec6ab134ad' move ignored: Not found: Peer with uuid f77c2a969bde45b49ac1bbbe242ff171 not found in consensus config
I20240502 01:59:07.723353 17304 rebalancer_tool.cc:756] re-synchronizing cluster state
W20240502 01:59:08.210459 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:08.210546 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0b29d58fe9c74d1cb4345bb6cfbd0535: could not find any suitable replica to move from server d4e5422f66454511ae5a50ec6ab134ad to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:08.210803 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:08.210903 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server 84987ad9e4a54d96af8569e4aaf0ed64 to server 7b66a8940fe54e21bec1b2db25b2fed5
W20240502 01:59:08.210951 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:08.211033 17304 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 9736e7c5365849759564e386d208932f: could not find any suitable replica to move from server d4e5422f66454511ae5a50ec6ab134ad to server ddae2a636423489693c12b7d7d166c09
I20240502 01:59:08.351018 17304 rebalancer_tool.cc:1074] tablet 3f938c5af51a43c9a23b753c4991e41d: 'ddae2a636423489693c12b7d7d166c09' -> '' move scheduled
I20240502 01:59:08.459081 17304 rebalancer_tool.cc:1074] tablet cd00409d47954ee2b7555e7ff09a6e28: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '7b66a8940fe54e21bec1b2db25b2fed5' move scheduled
I20240502 01:59:08.501700 17304 rebalancer_tool.cc:1136] tablet 3f938c5af51a43c9a23b753c4991e41d: 'ddae2a636423489693c12b7d7d166c09' -> '' move completed: OK
I20240502 01:59:08.566357 17304 rebalancer_tool.cc:1136] tablet 5804597cc507483dba622e6ec44e5ead: 'f77c2a969bde45b49ac1bbbe242ff171' -> '3b8d615c71e44ff887e0f037c3efcff7' move completed: OK
I20240502 01:59:08.669657 17304 rebalancer_tool.cc:1136] tablet d24986c2ebab46f086e3f1f5a53af26b: 'ddae2a636423489693c12b7d7d166c09' -> '84987ad9e4a54d96af8569e4aaf0ed64' move completed: OK
I20240502 01:59:08.681917 17304 rebalancer_tool.cc:1136] tablet 8f2fb504a49c44edac92fd935555c278: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'd4e5422f66454511ae5a50ec6ab134ad' move completed: OK
I20240502 01:59:08.689657 17304 rebalancer_tool.cc:1136] tablet ac5ec2c673cb4b988d1099e0442ec63f: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move completed: OK
I20240502 01:59:08.787124 17304 rebalancer_tool.cc:1136] tablet a901eb3063ea4911a822552db01d365d: 'f77c2a969bde45b49ac1bbbe242ff171' -> '3b8d615c71e44ff887e0f037c3efcff7' move completed: OK
I20240502 01:59:08.795972 17304 rebalancer_tool.cc:1136] tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532: 'ddae2a636423489693c12b7d7d166c09' -> '84987ad9e4a54d96af8569e4aaf0ed64' move completed: OK
I20240502 01:59:08.801685 17304 rebalancer_tool.cc:1136] tablet 30620f4774b8414eb4e27bd5ecb6ecee: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'd4e5422f66454511ae5a50ec6ab134ad' move completed: OK
I20240502 01:59:08.848421 17304 rebalancer_tool.cc:1136] tablet d1c832c721fb4b15bc6d266a1c295401: 'f77c2a969bde45b49ac1bbbe242ff171' -> '3b8d615c71e44ff887e0f037c3efcff7' move completed: Incomplete: tablet d1c832c721fb4b15bc6d266a1c295401, TS f77c2a969bde45b49ac1bbbe242ff171 -> TS 3b8d615c71e44ff887e0f037c3efcff7 move failed, target replica disappeared
I20240502 01:59:08.885879 17304 rebalancer_tool.cc:1136] tablet 6d5cffb7fcc54118861463e3b1fa4a8d: 'ddae2a636423489693c12b7d7d166c09' -> '84987ad9e4a54d96af8569e4aaf0ed64' move completed: OK
I20240502 01:59:08.894124 17304 rebalancer_tool.cc:1136] tablet 53edbd87d0d5471ca7626ca2071f75eb: 'f77c2a969bde45b49ac1bbbe242ff171' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move completed: OK
I20240502 01:59:08.908728 17304 rebalancer_tool.cc:1136] tablet 091d27d51acc4db0a1c7d20deeff5c10: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'd4e5422f66454511ae5a50ec6ab134ad' move completed: OK
I20240502 01:59:09.045904 17304 rebalancer_tool.cc:1136] tablet ce85785e670a4264b44797bb27cb4ecb: 'ddae2a636423489693c12b7d7d166c09' -> '3b8d615c71e44ff887e0f037c3efcff7' move completed: OK
I20240502 01:59:09.069080 17304 rebalancer_tool.cc:1084] tablet 81c8fe21ec7545e79450532b43a61ee7: '3b8d615c71e44ff887e0f037c3efcff7' -> 'f77c2a969bde45b49ac1bbbe242ff171' move ignored: Illegal state: tablet 81c8fe21ec7545e79450532b43a61ee7: replica f77c2a969bde45b49ac1bbbe242ff171 is already present
I20240502 01:59:09.069164 17304 rebalancer_tool.cc:756] re-synchronizing cluster state
Illegal state: tablet server 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): unacceptable health status UNAVAILABLE
@ 0x7f6b27e856ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f6b27e8703b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x561f590fd75c std::function<>::operator()() at ??:0
@ 0x7f6b27e74807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7f6b286b26db start_thread at ??:0
@ 0x7f6b25bea71f clone at ??:0
I20240502 01:59:09.440184 15059 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "8ddc239a297a46e48e069b985cb7b350"
dest_uuid: "ddae2a636423489693c12b7d7d166c09"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:57978
I20240502 01:59:09.440333 15059 raft_consensus.cc:595] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:59:09.447551 20303 ts_tablet_manager.cc:902] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:09.448043 20303 tablet_copy_client.cc:308] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:34323
I20240502 01:59:09.449635 15488 tablet_copy_service.cc:140] P 7b66a8940fe54e21bec1b2db25b2fed5: Received BeginTabletCopySession request for tablet 3f938c5af51a43c9a23b753c4991e41d from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:53377)
I20240502 01:59:09.449733 15488 tablet_copy_service.cc:161] P 7b66a8940fe54e21bec1b2db25b2fed5: Beginning new tablet copy session on tablet 3f938c5af51a43c9a23b753c4991e41d from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:53377: session id = 84987ad9e4a54d96af8569e4aaf0ed64-3f938c5af51a43c9a23b753c4991e41d
I20240502 01:59:09.450974 15488 tablet_copy_source_session.cc:205] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:09.451767 20303 ts_tablet_manager.cc:1875] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 01:59:09.453965 20303 ts_tablet_manager.cc:1888] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.9
I20240502 01:59:09.454123 20303 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3f938c5af51a43c9a23b753c4991e41d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:09.458042 20303 tablet_copy_client.cc:783] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:09.458218 20303 tablet_copy_client.cc:647] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:09.459920 18367 raft_consensus.cc:1050] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
I20240502 01:59:09.460265 20303 tablet_copy_client.cc:523] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:09.460318 18367 consensus_queue.cc:237] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:09.461555 20303 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
I20240502 01:59:09.461973 15059 raft_consensus.cc:1261] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:09.462388 17471 consensus_queue.cc:1035] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:09.464447 16840 raft_consensus.cc:1261] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:09.468482 15465 raft_consensus.cc:1261] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:09.469110 18319 raft_consensus.cc:979] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09: : Instructing follower 84987ad9e4a54d96af8569e4aaf0ed64 to start an election
I20240502 01:59:09.469177 18319 raft_consensus.cc:1067] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Signalling peer 84987ad9e4a54d96af8569e4aaf0ed64 to start an election
I20240502 01:59:09.469671 16838 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "8ddc239a297a46e48e069b985cb7b350"
dest_uuid: "84987ad9e4a54d96af8569e4aaf0ed64"
from {username='slave'} at 127.7.221.65:54083
I20240502 01:59:09.469775 16838 raft_consensus.cc:482] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 01:59:09.469818 16838 raft_consensus.cc:3037] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:59:09.470643 16838 raft_consensus.cc:504] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:09.471038 16838 leader_election.cc:288] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [CANDIDATE]: Term 2 election: Requested vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 01:59:09.471455 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8ddc239a297a46e48e069b985cb7b350" candidate_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:59:09.471554 15197 raft_consensus.cc:3037] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:59:09.472440 15197 raft_consensus.cc:2454] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84987ad9e4a54d96af8569e4aaf0ed64 in term 2.
I20240502 01:59:09.472736 16772 leader_election.cc:302] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 84987ad9e4a54d96af8569e4aaf0ed64; no voters:
I20240502 01:59:09.472883 19703 raft_consensus.cc:2781] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 01:59:09.472954 19703 raft_consensus.cc:686] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 2 LEADER]: Becoming Leader. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Running, Role: LEADER
I20240502 01:59:09.473088 19703 consensus_queue.cc:237] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:09.475009 15057 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8ddc239a297a46e48e069b985cb7b350" candidate_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "ddae2a636423489693c12b7d7d166c09"
I20240502 01:59:09.475131 15057 raft_consensus.cc:3032] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 01:59:09.475167 15057 raft_consensus.cc:729] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Running, Role: LEADER
I20240502 01:59:09.475327 15057 consensus_queue.cc:260] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:09.475498 15057 raft_consensus.cc:3037] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:59:09.476370 15057 raft_consensus.cc:2454] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84987ad9e4a54d96af8569e4aaf0ed64 in term 2.
I20240502 01:59:09.477985 14906 catalog_manager.cc:5484] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 reported cstate change: term changed from 1 to 2, leader changed from ddae2a636423489693c12b7d7d166c09 (127.7.221.65) to 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69). New cstate: current_term: 2 leader_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:09.478857 17910 consensus_queue.cc:1035] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:09.479421 15465 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "3f938c5af51a43c9a23b753c4991e41d"
dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:35160
I20240502 01:59:09.479537 15465 raft_consensus.cc:595] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:59:09.481510 17471 consensus_queue.cc:1035] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:09.491487 17471 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:09.493996 15465 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:09.494439 16840 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:09.496881 14909 catalog_manager.cc:5484] T f97c5c9bb4c84957857efc76d1c7717f P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 7 to 8, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:09.497207 20303 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=15 overwritten=0 applied=15 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:09.497473 20303 tablet_bootstrap.cc:492] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:09.497697 20303 ts_tablet_manager.cc:1367] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.036s user 0.026s sys 0.005s
I20240502 01:59:09.497750 15059 raft_consensus.cc:2932] T f97c5c9bb4c84957857efc76d1c7717f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:09.498172 20303 raft_consensus.cc:348] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:09.498291 20303 raft_consensus.cc:729] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:09.498816 20303 consensus_queue.cc:260] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 15, Last appended: 1.15, Last appended by leader: 15, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:09.499105 20303 ts_tablet_manager.cc:1398] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.001s user 0.004s sys 0.000s
I20240502 01:59:09.499694 15488 tablet_copy_service.cc:342] P 7b66a8940fe54e21bec1b2db25b2fed5: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-3f938c5af51a43c9a23b753c4991e41d received from {username='slave'} at 127.7.221.69:53377
I20240502 01:59:09.499768 15488 tablet_copy_service.cc:433] P 7b66a8940fe54e21bec1b2db25b2fed5: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-3f938c5af51a43c9a23b753c4991e41d on tablet 3f938c5af51a43c9a23b753c4991e41d with peer 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:09.520664 19909 ts_tablet_manager.cc:902] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:09.521162 19909 tablet_copy_client.cc:308] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:09.522737 15197 raft_consensus.cc:1203] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.9->[1.10-1.10] Dedup: 1.10->[]
I20240502 01:59:09.523219 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet a901eb3063ea4911a822552db01d365d from peer 7b66a8940fe54e21bec1b2db25b2fed5 ({username='slave'} at 127.7.221.68:41001)
I20240502 01:59:09.523306 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet a901eb3063ea4911a822552db01d365d from peer 7b66a8940fe54e21bec1b2db25b2fed5 at {username='slave'} at 127.7.221.68:41001: session id = 7b66a8940fe54e21bec1b2db25b2fed5-a901eb3063ea4911a822552db01d365d
I20240502 01:59:09.524235 15216 tablet_copy_source_session.cc:205] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:09.525005 19909 ts_tablet_manager.cc:1875] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 01:59:09.527213 19909 ts_tablet_manager.cc:1888] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.8
I20240502 01:59:09.527396 19909 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a901eb3063ea4911a822552db01d365d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:09.528381 19909 tablet_copy_client.cc:783] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:09.528520 19909 tablet_copy_client.cc:647] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:09.529645 19909 tablet_copy_client.cc:523] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:09.530928 19909 tablet_bootstrap.cc:492] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:59:09.561518 19909 tablet_bootstrap.cc:492] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap replayed 1/1 log segments. Stats: ops{read=10 overwritten=0 applied=9 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 1 replicates
I20240502 01:59:09.561900 19909 tablet_bootstrap.cc:492] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap complete.
I20240502 01:59:09.562108 19909 ts_tablet_manager.cc:1367] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.031s user 0.025s sys 0.000s
I20240502 01:59:09.563815 19909 raft_consensus.cc:348] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 NON_PARTICIPANT]: Replica starting. Triggering 1 pending ops. Active config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 01:59:09.564039 19909 raft_consensus.cc:729] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: LEARNER
I20240502 01:59:09.564138 19909 consensus_queue.cc:260] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 9, Last appended: 1.10, Last appended by leader: 10, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } }
I20240502 01:59:09.565084 19909 ts_tablet_manager.cc:1398] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.003s user 0.005s sys 0.000s
I20240502 01:59:09.565557 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session 7b66a8940fe54e21bec1b2db25b2fed5-a901eb3063ea4911a822552db01d365d received from {username='slave'} at 127.7.221.68:41001
I20240502 01:59:09.565635 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session 7b66a8940fe54e21bec1b2db25b2fed5-a901eb3063ea4911a822552db01d365d on tablet a901eb3063ea4911a822552db01d365d with peer 7b66a8940fe54e21bec1b2db25b2fed5
W20240502 01:59:09.579280 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.583282 15328 consensus_queue.cc:237] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
W20240502 01:59:09.585527 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.585950 15059 raft_consensus.cc:1261] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:09.586007 16840 raft_consensus.cc:1261] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:09.586424 20213 consensus_queue.cc:1035] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:09.586658 18071 consensus_queue.cc:1035] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:09.588786 20213 raft_consensus.cc:2932] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:09.589586 15059 raft_consensus.cc:2932] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:09.590749 16840 raft_consensus.cc:2932] T 9c2892024ce942e1bf1834c1b9ad2049 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:09.592213 14907 catalog_manager.cc:5484] T 9c2892024ce942e1bf1834c1b9ad2049 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 6 to 7, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } } }
I20240502 01:59:09.602969 15328 consensus_queue.cc:237] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:09.604430 15059 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
W20240502 01:59:09.604900 15266 consensus_peers.cc:485] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.605194 20213 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
W20240502 01:59:09.605767 15265 consensus_peers.cc:485] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.607400 18071 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:09.608070 15059 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 01:59:09.610670 14909 catalog_manager.cc:5484] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 8 to 9, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:09.622931 15328 consensus_queue.cc:237] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } }
I20240502 01:59:09.624456 15197 raft_consensus.cc:1261] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 01:59:09.624866 15265 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): Couldn't send request to peer 7b66a8940fe54e21bec1b2db25b2fed5. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6db29f2516e1486782be0312c9bcac8b. This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.625541 18416 consensus_queue.cc:1035] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:09.635460 18416 raft_consensus.cc:2932] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } }
I20240502 01:59:09.639096 14907 catalog_manager.cc:5484] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 6 to 7, NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:09.640902 15197 raft_consensus.cc:2932] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } }
I20240502 01:59:09.648218 20423 ts_tablet_manager.cc:902] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:09.649030 20423 tablet_copy_client.cc:308] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:09.649703 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet 21e2e55d1a0d4b6eb18b4ade9468607f from peer ddae2a636423489693c12b7d7d166c09 ({username='slave'} at 127.7.221.65:40827)
I20240502 01:59:09.649837 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet 21e2e55d1a0d4b6eb18b4ade9468607f from peer ddae2a636423489693c12b7d7d166c09 at {username='slave'} at 127.7.221.65:40827: session id = ddae2a636423489693c12b7d7d166c09-21e2e55d1a0d4b6eb18b4ade9468607f
I20240502 01:59:09.651324 15057 consensus_queue.cc:237] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:09.652628 15216 tablet_copy_source_session.cc:205] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:09.653158 15465 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 01:59:09.653517 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.653733 18150 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:09.654497 20423 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 21e2e55d1a0d4b6eb18b4ade9468607f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:09.654547 15197 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:09.654733 16840 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:09.655180 18319 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:09.655568 18319 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:09.655928 20429 raft_consensus.cc:1013] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:09.656102 18367 raft_consensus.cc:1050] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER f77c2a969bde45b49ac1bbbe242ff171 to VOTER
I20240502 01:59:09.656468 18367 consensus_queue.cc:237] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: false } }
I20240502 01:59:09.657112 20423 tablet_copy_client.cc:783] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:09.657301 20423 tablet_copy_client.cc:647] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:09.658847 20423 tablet_copy_client.cc:523] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:09.658893 15328 raft_consensus.cc:1261] T 81c8fe21ec7545e79450532b43a61ee7 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:09.658928 20429 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:09.659353 15197 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:09.659607 17910 consensus_queue.cc:1035] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:09.660418 20423 tablet_bootstrap.cc:492] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 01:59:09.661557 14907 catalog_manager.cc:5484] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:09.662087 15465 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
W20240502 01:59:09.663014 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.663121 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.663928 18150 raft_consensus.cc:1050] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09: attempting to promote NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 to VOTER
I20240502 01:59:09.664319 18150 consensus_queue.cc:237] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:09.666275 16840 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:09.668066 16838 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:09.669427 15197 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:09.669425 18319 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:09.669826 15465 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:09.669872 18319 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
W20240502 01:59:09.669974 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.670933 20429 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:09.674487 18150 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:09.676489 15465 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:09.680056 14909 catalog_manager.cc:5484] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 8 to 9, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:09.681275 16838 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:09.683946 15197 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:09.684327 15057 consensus_queue.cc:237] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:09.685832 15465 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:09.687542 20437 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:09.687759 16840 raft_consensus.cc:1261] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:09.689158 20438 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:09.691407 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 77ae087802144c45924c1b79f561afe8 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:09.691840 20429 consensus_queue.cc:1035] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
W20240502 01:59:09.692095 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.692927 20423 tablet_bootstrap.cc:492] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:09.692965 14909 catalog_manager.cc:5484] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 9 to 10, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:09.692997 16840 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:09.693413 20423 tablet_bootstrap.cc:492] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09: Bootstrap complete.
I20240502 01:59:09.693639 20423 ts_tablet_manager.cc:1367] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.033s user 0.032s sys 0.000s
I20240502 01:59:09.694159 20423 raft_consensus.cc:348] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:09.694371 20423 raft_consensus.cc:729] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: LEARNER
I20240502 01:59:09.694530 20423 consensus_queue.cc:260] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 1.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:09.695101 20423 ts_tablet_manager.cc:1398] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 01:59:09.695647 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session ddae2a636423489693c12b7d7d166c09-21e2e55d1a0d4b6eb18b4ade9468607f received from {username='slave'} at 127.7.221.65:40827
I20240502 01:59:09.695732 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session ddae2a636423489693c12b7d7d166c09-21e2e55d1a0d4b6eb18b4ade9468607f on tablet 21e2e55d1a0d4b6eb18b4ade9468607f with peer ddae2a636423489693c12b7d7d166c09
I20240502 01:59:09.696775 15174 tablet_service.cc:1514] Processing DeleteTablet for tablet 77ae087802144c45924c1b79f561afe8 with delete_type TABLET_DATA_TOMBSTONED (TS 26a3d5ced97d4dcc919b80fde53f6a5b not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:37206
I20240502 01:59:09.696959 15465 raft_consensus.cc:2932] T 77ae087802144c45924c1b79f561afe8 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:09.697678 20445 tablet_replica.cc:330] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b: stopping tablet replica
I20240502 01:59:09.697885 20445 raft_consensus.cc:2229] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:09.698210 20445 raft_consensus.cc:2258] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:09.699051 20445 ts_tablet_manager.cc:1875] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:09.704109 20445 ts_tablet_manager.cc:1888] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 01:59:09.704249 20445 log.cc:1196] T 77ae087802144c45924c1b79f561afe8 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/77ae087802144c45924c1b79f561afe8
I20240502 01:59:09.704917 14900 catalog_manager.cc:4830] TS 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905): tablet 77ae087802144c45924c1b79f561afe8 (table rebalance_test_table_7 [id=1fafbde99fe04ac28cefeef0570a2828]) successfully deleted
W20240502 01:59:09.722123 16771 connection.cc:535] client connection to 127.7.221.71:43207 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 01:59:09.722348 15263 connection.cc:535] client connection to 127.7.221.71:43207 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 01:59:09.722395 15396 connection.cc:535] client connection to 127.7.221.71:43207 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 01:59:09.722551 14992 connection.cc:535] client connection to 127.7.221.71:43207 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 01:59:09.723183 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.723459 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:09.723531 15396 consensus_peers.cc:485] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.723580 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.723778 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.724059 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.724115 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.741901 15465 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846"
dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:35180
I20240502 01:59:09.742045 15465 raft_consensus.cc:595] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:59:09.759347 15197 raft_consensus.cc:1261] T 856f623374824d94913db0cb9faa25d1 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 2 FOLLOWER]: Refusing update from remote peer 84987ad9e4a54d96af8569e4aaf0ed64: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
I20240502 01:59:09.759845 20110 consensus_queue.cc:1035] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 01:59:09.761454 18279 raft_consensus.cc:979] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5: : Instructing follower f77c2a969bde45b49ac1bbbe242ff171 to start an election
I20240502 01:59:09.761555 19073 raft_consensus.cc:1067] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Signalling peer f77c2a969bde45b49ac1bbbe242ff171 to start an election
I20240502 01:59:09.762216 15328 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846"
dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171"
from {username='slave'} at 127.7.221.68:60473
I20240502 01:59:09.762337 15328 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 01:59:09.762395 15328 raft_consensus.cc:3037] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:59:09.763523 15328 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:09.763967 15328 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 2 election: Requested vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 01:59:09.764199 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:59:09.764330 15465 raft_consensus.cc:3032] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 01:59:09.764395 15465 raft_consensus.cc:729] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Running, Role: LEADER
I20240502 01:59:09.764570 15465 consensus_queue.cc:260] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:09.764714 15465 raft_consensus.cc:3037] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Advancing to term 2
W20240502 01:59:09.765436 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:09.765767 15465 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 01:59:09.765872 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:09.765938 15263 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 4 voters: 1 yes votes; 2 no votes. yes voters: f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:09.766072 20213 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
I20240502 01:59:09.770891 18109 raft_consensus.cc:1050] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 to VOTER
I20240502 01:59:09.771236 18109 consensus_queue.cc:237] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } }
I20240502 01:59:09.772717 15328 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:09.773288 18367 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:09.773370 15328 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 2 index: 11. (index mismatch)
W20240502 01:59:09.773439 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.774806 15465 raft_consensus.cc:1261] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:09.775142 20437 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:09.775260 18367 consensus_queue.cc:1035] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
W20240502 01:59:09.775760 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.780001 19214 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } }
I20240502 01:59:09.780449 15328 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } }
I20240502 01:59:09.782531 15465 raft_consensus.cc:2932] T cd00409d47954ee2b7555e7ff09a6e28 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } }
I20240502 01:59:09.783746 14909 catalog_manager.cc:5484] T cd00409d47954ee2b7555e7ff09a6e28 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 10 to 11, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } }
I20240502 01:59:09.791357 19909 ts_tablet_manager.cc:902] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 01:59:09.791857 19909 tablet_copy_client.cc:308] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 01:59:09.792289 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet d24986c2ebab46f086e3f1f5a53af26b from peer 7b66a8940fe54e21bec1b2db25b2fed5 ({username='slave'} at 127.7.221.68:41001)
I20240502 01:59:09.792385 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet d24986c2ebab46f086e3f1f5a53af26b from peer 7b66a8940fe54e21bec1b2db25b2fed5 at {username='slave'} at 127.7.221.68:41001: session id = 7b66a8940fe54e21bec1b2db25b2fed5-d24986c2ebab46f086e3f1f5a53af26b
I20240502 01:59:09.793212 15216 tablet_copy_source_session.cc:205] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:09.793833 19909 ts_tablet_manager.cc:1875] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 01:59:09.796051 19909 ts_tablet_manager.cc:1888] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240502 01:59:09.796207 19909 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d24986c2ebab46f086e3f1f5a53af26b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:09.797171 19909 tablet_copy_client.cc:783] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:09.797559 19909 tablet_copy_client.cc:647] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:09.798924 19909 tablet_copy_client.cc:523] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:09.799944 19909 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:59:09.813637 15465 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "3f938c5af51a43c9a23b753c4991e41d"
dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:35196
I20240502 01:59:09.813757 15465 raft_consensus.cc:595] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:59:09.820727 19909 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:09.821002 19909 tablet_bootstrap.cc:492] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap complete.
I20240502 01:59:09.821171 19909 ts_tablet_manager.cc:1367] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.021s user 0.015s sys 0.004s
I20240502 01:59:09.821563 19909 raft_consensus.cc:348] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } }
I20240502 01:59:09.821671 19909 raft_consensus.cc:729] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: LEARNER
I20240502 01:59:09.821782 19909 consensus_queue.cc:260] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } }
I20240502 01:59:09.822178 19909 ts_tablet_manager.cc:1398] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.003s sys 0.000s
I20240502 01:59:09.822734 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session 7b66a8940fe54e21bec1b2db25b2fed5-d24986c2ebab46f086e3f1f5a53af26b received from {username='slave'} at 127.7.221.68:41001
I20240502 01:59:09.822813 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session 7b66a8940fe54e21bec1b2db25b2fed5-d24986c2ebab46f086e3f1f5a53af26b on tablet d24986c2ebab46f086e3f1f5a53af26b with peer 7b66a8940fe54e21bec1b2db25b2fed5
I20240502 01:59:09.833115 15197 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 2 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 2 index: 11. (index mismatch)
I20240502 01:59:09.833478 20437 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:09.835907 16840 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 2 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 2 index: 11. (index mismatch)
I20240502 01:59:09.837174 18150 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
W20240502 01:59:09.840431 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.841041 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:09.871421 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.874487 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:09.886782 15127 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.892256 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.906494 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:09.929864 20437 raft_consensus.cc:1050] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09: attempting to promote NON_VOTER 26a3d5ced97d4dcc919b80fde53f6a5b to VOTER
W20240502 01:59:09.929966 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.930269 20437 consensus_queue.cc:237] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: false } }
I20240502 01:59:09.931670 15197 raft_consensus.cc:1261] T 1eb15147ff6c4d0b83b12dec736fc576 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEARNER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:09.932060 20438 consensus_queue.cc:1035] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
W20240502 01:59:09.932168 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.932258 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:09.955309 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:09.991058 15197 raft_consensus.cc:1261] T 8ddc239a297a46e48e069b985cb7b350 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 2 FOLLOWER]: Refusing update from remote peer 84987ad9e4a54d96af8569e4aaf0ed64: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 01:59:09.991487 19703 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:09.993320 20303 ts_tablet_manager.cc:902] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:09.993774 20303 tablet_copy_client.cc:308] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 01:59:09.994225 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet 091d27d51acc4db0a1c7d20deeff5c10 from peer 84987ad9e4a54d96af8569e4aaf0ed64 ({username='slave'} at 127.7.221.69:51343)
I20240502 01:59:09.994331 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet 091d27d51acc4db0a1c7d20deeff5c10 from peer 84987ad9e4a54d96af8569e4aaf0ed64 at {username='slave'} at 127.7.221.69:51343: session id = 84987ad9e4a54d96af8569e4aaf0ed64-091d27d51acc4db0a1c7d20deeff5c10
I20240502 01:59:09.995268 15351 tablet_copy_source_session.cc:205] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:09.996052 20303 ts_tablet_manager.cc:1875] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 01:59:09.996098 15057 raft_consensus.cc:1261] T 8ddc239a297a46e48e069b985cb7b350 P ddae2a636423489693c12b7d7d166c09 [term 2 FOLLOWER]: Refusing update from remote peer 84987ad9e4a54d96af8569e4aaf0ed64: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 01:59:09.997028 20110 consensus_queue.cc:1035] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:09.998374 20303 ts_tablet_manager.cc:1888] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240502 01:59:09.998541 20303 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 091d27d51acc4db0a1c7d20deeff5c10. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:09.999606 20303 tablet_copy_client.cc:783] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:10.000499 20303 tablet_copy_client.cc:647] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:10.001822 20303 tablet_copy_client.cc:523] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:10.003015 20303 tablet_bootstrap.cc:492] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap starting.
I20240502 01:59:10.023314 20303 tablet_bootstrap.cc:492] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap replayed 1/1 log segments. Stats: ops{read=9 overwritten=0 applied=9 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:10.023610 20303 tablet_bootstrap.cc:492] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: Bootstrap complete.
I20240502 01:59:10.023865 20303 ts_tablet_manager.cc:1367] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent bootstrapping tablet: real 0.021s user 0.014s sys 0.006s
I20240502 01:59:10.024446 20303 raft_consensus.cc:348] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:10.024583 20303 raft_consensus.cc:729] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Initialized, Role: LEARNER
I20240502 01:59:10.024667 20303 consensus_queue.cc:260] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 9, Last appended: 1.9, Last appended by leader: 9, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:10.024920 20303 ts_tablet_manager.cc:1398] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:10.025390 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-091d27d51acc4db0a1c7d20deeff5c10 received from {username='slave'} at 127.7.221.69:51343
I20240502 01:59:10.025477 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session 84987ad9e4a54d96af8569e4aaf0ed64-091d27d51acc4db0a1c7d20deeff5c10 on tablet 091d27d51acc4db0a1c7d20deeff5c10 with peer 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:10.050159 15465 raft_consensus.cc:1203] T a901eb3063ea4911a822552db01d365d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.9->[1.10-1.10] Dedup: 1.10->[]
I20240502 01:59:10.054481 16840 raft_consensus.cc:1203] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.14->[1.15-1.15] Dedup: 1.15->[]
I20240502 01:59:10.084945 15465 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "3f938c5af51a43c9a23b753c4991e41d"
dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:35220
I20240502 01:59:10.085086 15465 raft_consensus.cc:595] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:59:10.120224 15057 raft_consensus.cc:1203] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.5->[1.6-1.6] Dedup: 1.6->[]
W20240502 01:59:10.178401 14901 catalog_manager.cc:4573] TS 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): ChangeConfig:ADD_PEER:NON_VOTER RPC failed for tablet 3f938c5af51a43c9a23b753c4991e41d: Remote error: Service unavailable: leader transfer in progress
I20240502 01:59:10.203040 19909 ts_tablet_manager.cc:902] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 01:59:10.203480 19909 tablet_copy_client.cc:308] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 01:59:10.203894 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet 6db29f2516e1486782be0312c9bcac8b from peer 7b66a8940fe54e21bec1b2db25b2fed5 ({username='slave'} at 127.7.221.68:60473)
I20240502 01:59:10.203982 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet 6db29f2516e1486782be0312c9bcac8b from peer 7b66a8940fe54e21bec1b2db25b2fed5 at {username='slave'} at 127.7.221.68:60473: session id = 7b66a8940fe54e21bec1b2db25b2fed5-6db29f2516e1486782be0312c9bcac8b
I20240502 01:59:10.204764 15351 tablet_copy_source_session.cc:205] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 01:59:10.205530 19909 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6db29f2516e1486782be0312c9bcac8b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 01:59:10.207404 19909 tablet_copy_client.cc:783] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Starting download of 0 data blocks...
I20240502 01:59:10.207552 19909 tablet_copy_client.cc:647] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Starting download of 1 WAL segments...
I20240502 01:59:10.208619 19909 tablet_copy_client.cc:523] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 01:59:10.209640 19909 tablet_bootstrap.cc:492] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap starting.
I20240502 01:59:10.228453 19909 tablet_bootstrap.cc:492] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 01:59:10.228709 19909 tablet_bootstrap.cc:492] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5: Bootstrap complete.
I20240502 01:59:10.228860 19909 ts_tablet_manager.cc:1367] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent bootstrapping tablet: real 0.019s user 0.018s sys 0.000s
I20240502 01:59:10.229209 19909 raft_consensus.cc:348] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } }
I20240502 01:59:10.229321 19909 raft_consensus.cc:729] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7b66a8940fe54e21bec1b2db25b2fed5, State: Initialized, Role: LEARNER
I20240502 01:59:10.229405 19909 consensus_queue.cc:260] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } }
I20240502 01:59:10.229629 19909 ts_tablet_manager.cc:1398] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 01:59:10.230146 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session 7b66a8940fe54e21bec1b2db25b2fed5-6db29f2516e1486782be0312c9bcac8b received from {username='slave'} at 127.7.221.68:60473
I20240502 01:59:10.230242 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session 7b66a8940fe54e21bec1b2db25b2fed5-6db29f2516e1486782be0312c9bcac8b on tablet 6db29f2516e1486782be0312c9bcac8b with peer 7b66a8940fe54e21bec1b2db25b2fed5
I20240502 01:59:10.285503 15465 raft_consensus.cc:1203] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240502 01:59:10.342685 15465 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "3f938c5af51a43c9a23b753c4991e41d"
dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:35248
I20240502 01:59:10.342818 15465 raft_consensus.cc:595] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:59:10.369537 16840 raft_consensus.cc:1203] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.8->[1.9-1.9] Dedup: 1.9->[]
W20240502 01:59:10.438683 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:10.439785 15057 consensus_queue.cc:237] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 2.11, Last appended by leader: 10, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:10.441377 15328 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 2 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
I20240502 01:59:10.441412 16840 raft_consensus.cc:1261] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 2 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 2 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
I20240502 01:59:10.441802 18150 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 01:59:10.442060 18150 consensus_queue.cc:1035] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
W20240502 01:59:10.442452 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:10.443338 18319 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 [term 2 LEADER]: Committing config change with OpId 2.12: config changed from index 10 to 12, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:10.443626 16840 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 2 FOLLOWER]: Committing config change with OpId 2.12: config changed from index 10 to 12, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:10.445801 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6c4e85e7050b4dc8a4ce6bba818854a2 with cas_config_opid_index 10: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
W20240502 01:59:10.446168 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 01:59:10.446314 15328 raft_consensus.cc:2932] T 6c4e85e7050b4dc8a4ce6bba818854a2 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Committing config change with OpId 2.12: config changed from index 10 to 12, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:10.446355 14909 catalog_manager.cc:5484] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 10 to 12, VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) evicted. New cstate: current_term: 2 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:10.449436 15174 tablet_service.cc:1514] Processing DeleteTablet for tablet 6c4e85e7050b4dc8a4ce6bba818854a2 with delete_type TABLET_DATA_TOMBSTONED (TS 26a3d5ced97d4dcc919b80fde53f6a5b not found in new config with opid_index 12) from {username='slave'} at 127.0.0.1:37206
I20240502 01:59:10.454793 20833 tablet_replica.cc:330] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b: stopping tablet replica
I20240502 01:59:10.454931 20833 raft_consensus.cc:2229] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:10.455072 20833 raft_consensus.cc:2258] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:10.455581 20833 ts_tablet_manager.cc:1875] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:10.457357 20833 ts_tablet_manager.cc:1888] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.11
I20240502 01:59:10.457429 20833 log.cc:1196] T 6c4e85e7050b4dc8a4ce6bba818854a2 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/6c4e85e7050b4dc8a4ce6bba818854a2
I20240502 01:59:10.457940 14900 catalog_manager.cc:4830] TS 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905): tablet 6c4e85e7050b4dc8a4ce6bba818854a2 (table rebalance_test_table_9 [id=12ee6dc62f814bd7b3c13bee4495d9e0]) successfully deleted
W20240502 01:59:10.503815 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:10.536026 17910 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:10.558964 18279 raft_consensus.cc:1050] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
W20240502 01:59:10.559077 18279 raft_consensus.cc:1053] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: Unable to promote non-voter 84987ad9e4a54d96af8569e4aaf0ed64: Service unavailable: leader transfer in progress
I20240502 01:59:10.599603 15465 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "3f938c5af51a43c9a23b753c4991e41d"
dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:35282
I20240502 01:59:10.599735 15465 raft_consensus.cc:595] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:59:10.640563 17910 raft_consensus.cc:1050] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER ddae2a636423489693c12b7d7d166c09 to VOTER
I20240502 01:59:10.640944 17910 consensus_queue.cc:237] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:10.642326 15328 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:10.642813 20953 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 01:59:10.643013 15057 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:10.643379 20953 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 01:59:10.643657 15130 consensus_peers.cc:485] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:10.644073 15465 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 01:59:10.644520 20953 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 01:59:10.644634 15131 consensus_peers.cc:485] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:10.645218 20953 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, ddae2a636423489693c12b7d7d166c09 (127.7.221.65) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:10.645710 15057 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, ddae2a636423489693c12b7d7d166c09 (127.7.221.65) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:10.646888 15465 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, ddae2a636423489693c12b7d7d166c09 (127.7.221.65) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:10.649693 14906 catalog_manager.cc:5484] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 6 to 7, ddae2a636423489693c12b7d7d166c09 (127.7.221.65) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:10.650684 15328 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, ddae2a636423489693c12b7d7d166c09 (127.7.221.65) changed from NON_VOTER to VOTER. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:10.652715 15197 consensus_queue.cc:237] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } }
I20240502 01:59:10.654079 15057 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:10.654377 15328 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:10.654560 20953 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:10.654649 15465 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:10.654742 20954 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:10.655017 20953 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240502 01:59:10.655562 15131 consensus_peers.cc:485] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:10.656342 20954 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:10.657058 15057 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:10.657405 15465 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:10.658353 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 21e2e55d1a0d4b6eb18b4ade9468607f with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:10.658552 15328 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } }
I20240502 01:59:10.659487 14907 catalog_manager.cc:5484] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 7 to 8, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 01:59:10.663671 15197 consensus_queue.cc:237] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } }
I20240502 01:59:10.664849 15057 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:10.665377 20953 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:10.665974 15465 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:10.666065 15328 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 01:59:10.666343 18109 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:10.666651 20953 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 01:59:10.668149 18367 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } }
I20240502 01:59:10.668583 15464 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } }
I20240502 01:59:10.668733 15328 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } }
I20240502 01:59:10.670068 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 21e2e55d1a0d4b6eb18b4ade9468607f with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:10.670809 15057 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } }
I20240502 01:59:10.671237 14907 catalog_manager.cc:5484] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 8 to 9, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } }
I20240502 01:59:10.674877 15197 consensus_queue.cc:237] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } }
I20240502 01:59:10.675920 15464 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:10.675997 15057 raft_consensus.cc:1261] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:10.676273 17910 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:10.676793 17910 consensus_queue.cc:1035] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:10.677819 18109 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } }
I20240502 01:59:10.678244 15464 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } }
I20240502 01:59:10.678570 15057 raft_consensus.cc:2932] T 21e2e55d1a0d4b6eb18b4ade9468607f P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } } }
I20240502 01:59:10.679492 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 21e2e55d1a0d4b6eb18b4ade9468607f with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 01:59:10.680142 14909 catalog_manager.cc:5484] T 21e2e55d1a0d4b6eb18b4ade9468607f P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 9 to 10, VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:10.682772 15309 tablet_service.cc:1514] Processing DeleteTablet for tablet 21e2e55d1a0d4b6eb18b4ade9468607f with delete_type TABLET_DATA_TOMBSTONED (TS f77c2a969bde45b49ac1bbbe242ff171 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:43766
I20240502 01:59:10.683145 20961 tablet_replica.cc:330] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171: stopping tablet replica
I20240502 01:59:10.683312 20961 raft_consensus.cc:2229] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 01:59:10.683552 20961 raft_consensus.cc:2258] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 01:59:10.684247 20961 ts_tablet_manager.cc:1875] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 01:59:10.685966 20961 ts_tablet_manager.cc:1888] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 01:59:10.686028 20961 log.cc:1196] T 21e2e55d1a0d4b6eb18b4ade9468607f P f77c2a969bde45b49ac1bbbe242ff171: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/21e2e55d1a0d4b6eb18b4ade9468607f
I20240502 01:59:10.686491 14900 catalog_manager.cc:4830] TS f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): tablet 21e2e55d1a0d4b6eb18b4ade9468607f (table rebalance_test_table_0 [id=9736e7c5365849759564e386d208932f]) successfully deleted
I20240502 01:59:10.689414 15464 raft_consensus.cc:1203] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 01:59:10.812647 20954 raft_consensus.cc:1050] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b: attempting to promote NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 to VOTER
I20240502 01:59:10.813091 20954 consensus_queue.cc:237] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } }
I20240502 01:59:10.814989 15464 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 01:59:10.815410 16840 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 01:59:10.815603 20953 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 01:59:10.815912 20953 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
W20240502 01:59:10.816229 15127 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:10.816937 15131 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:10.818727 20953 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } }
I20240502 01:59:10.819301 16840 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } }
I20240502 01:59:10.821209 15464 raft_consensus.cc:2932] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } }
I20240502 01:59:10.821624 14912 catalog_manager.cc:5484] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 11 to 12, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
W20240502 01:59:10.853394 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:10.863868 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 01:59:10.867897 15464 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "3f938c5af51a43c9a23b753c4991e41d"
dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:35298
I20240502 01:59:10.868052 15464 raft_consensus.cc:595] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Received request to transfer leadership
I20240502 01:59:10.879973 15197 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "d24986c2ebab46f086e3f1f5a53af26b"
dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:36556
I20240502 01:59:10.880106 15197 raft_consensus.cc:595] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Received request to transfer leadership
I20240502 01:59:10.919509 18416 raft_consensus.cc:1050] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
I20240502 01:59:10.919926 18416 consensus_queue.cc:237] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:10.921505 15057 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:10.921799 16840 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 01:59:10.922197 21147 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
W20240502 01:59:10.922554 15265 consensus_peers.cc:485] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:10.922715 20213 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 01:59:10.927022 20213 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:10.929199 15057 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:10.929186 16838 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:10.931553 14906 catalog_manager.cc:5484] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 9 to 10, 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:10.935895 15328 consensus_queue.cc:237] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:10.937172 16838 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:10.937502 15057 raft_consensus.cc:1261] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 01:59:10.937527 21147 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:10.937882 21147 consensus_queue.cc:1035] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 01:59:10.939700 21147 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:10.940721 16838 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:10.941205 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 091d27d51acc4db0a1c7d20deeff5c10 with cas_config_opid_index 10: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
W20240502 01:59:10.941694 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:10.942129 15057 raft_consensus.cc:2932] T 091d27d51acc4db0a1c7d20deeff5c10 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 01:59:10.943559 14912 catalog_manager.cc:5484] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 10 to 11, VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 01:59:10.945859 18109 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:10.987951 15444 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 01:59:10.988646 15174 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:10.989533 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 01:59:10.990190 16820 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 01:59:11.003809 15309 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
W20240502 01:59:11.035452 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:11.038491 17297 rebalancer_tool-test.cc:1260] rebalancer run info: Runtime error: /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu: process exited with non-zero status 1
stdout: Per-server replica distribution summary:
Statistic | Value
-----------------------+-----------
Minimum Replica Count | 0
Maximum Replica Count | 23
Average Replica Count | 11.250000
Per-table replica distribution summary:
Replica Skew | Value
--------------+----------
Minimum | 3
Maximum | 3
Average | 3.000000
stderr: W20240502 01:59:03.136802 17301 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:59:03.137086 17301 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:59:03.141773 17301 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 01:59:04.161275 17301 rebalancer_tool.cc:215] running whole-cluster rebalancing
W20240502 01:59:04.679430 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.679570 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:04.679625 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.679694 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0b29d58fe9c74d1cb4345bb6cfbd0535: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.679755 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:04.679930 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.680068 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.680137 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 9736e7c5365849759564e386d208932f: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:04.680207 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.680248 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.680292 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.680346 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 9736e7c5365849759564e386d208932f: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.680395 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.680441 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:04.696787 17301 rebalancer_tool.cc:1084] tablet e2f4ca025890492880608c7975650969: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move ignored: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } }.
Pending config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 01:59:04.696976 17301 rebalancer_tool.cc:756] re-synchronizing cluster state
W20240502 01:59:05.002545 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:05.002677 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.002750 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:05.002789 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.002858 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.002892 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:05.002928 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:05.002964 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:05.003000 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.003036 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0b29d58fe9c74d1cb4345bb6cfbd0535: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:05.003090 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:05.003124 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.003156 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:05.003190 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:05.003226 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:05.003262 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.003295 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:05.054206 17301 rebalancer_tool.cc:1074] tablet e00d1ed5cde044cb9bdebf787590261a: 'f77c2a969bde45b49ac1bbbe242ff171' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move scheduled
I20240502 01:59:05.099785 17301 rebalancer_tool.cc:1074] tablet 856f623374824d94913db0cb9faa25d1: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:05.146085 17301 rebalancer_tool.cc:1074] tablet 8ddc239a297a46e48e069b985cb7b350: 'ddae2a636423489693c12b7d7d166c09' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:05.161274 17301 rebalancer_tool.cc:1084] tablet 30620f4774b8414eb4e27bd5ecb6ecee: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'd4e5422f66454511ae5a50ec6ab134ad' move ignored: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }.
Pending config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: true } }
I20240502 01:59:05.161356 17301 rebalancer_tool.cc:756] re-synchronizing cluster state
W20240502 01:59:05.733759 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:05.733979 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.734081 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:05.734118 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:05.734148 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.734246 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:05.734297 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:05.734494 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:05.734527 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.734551 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:05.951186 17301 rebalancer_tool.cc:1074] tablet 5804597cc507483dba622e6ec44e5ead: 'f77c2a969bde45b49ac1bbbe242ff171' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:06.098896 17301 rebalancer_tool.cc:1074] tablet 091d27d51acc4db0a1c7d20deeff5c10: '7b66a8940fe54e21bec1b2db25b2fed5' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:06.259925 17301 rebalancer_tool.cc:1074] tablet cd00409d47954ee2b7555e7ff09a6e28: 'ddae2a636423489693c12b7d7d166c09' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move scheduled
I20240502 01:59:06.384634 17301 rebalancer_tool.cc:1074] tablet f97c5c9bb4c84957857efc76d1c7717f: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:06.422777 17301 rebalancer_tool.cc:1074] tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:06.470039 17301 rebalancer_tool.cc:1074] tablet 3f938c5af51a43c9a23b753c4991e41d: '7b66a8940fe54e21bec1b2db25b2fed5' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:06.550428 17301 rebalancer_tool.cc:1074] tablet ac5ec2c673cb4b988d1099e0442ec63f: 'f77c2a969bde45b49ac1bbbe242ff171' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:06.830618 17301 rebalancer_tool.cc:1074] tablet 6db29f2516e1486782be0312c9bcac8b: 'ddae2a636423489693c12b7d7d166c09' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move scheduled
I20240502 01:59:07.144243 17301 rebalancer_tool.cc:1074] tablet 490e37fac0844a08a07a4b281bc33bf9: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:07.304280 17301 rebalancer_tool.cc:1074] tablet 30620f4774b8414eb4e27bd5ecb6ecee: '7b66a8940fe54e21bec1b2db25b2fed5' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:07.619251 17301 rebalancer_tool.cc:1084] tablet a901eb3063ea4911a822552db01d365d: 'f77c2a969bde45b49ac1bbbe242ff171' -> '84987ad9e4a54d96af8569e4aaf0ed64' move ignored: Not found: Peer with uuid f77c2a969bde45b49ac1bbbe242ff171 not found in consensus config
I20240502 01:59:07.619320 17301 rebalancer_tool.cc:756] re-synchronizing cluster state
W20240502 01:59:08.183509 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:08.203783 17301 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0b29d58fe9c74d1cb4345bb6cfbd0535: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 3b8d615c71e44ff887e0f037c3efcff7
I20240502 01:59:08.235075 17301 rebalancer_tool.cc:1074] tablet 81c8fe21ec7545e79450532b43a61ee7: 'd4e5422f66454511ae5a50ec6ab134ad' -> 'f77c2a969bde45b49ac1bbbe242ff171' move scheduled
I20240502 01:59:08.434267 17301 rebalancer_tool.cc:1074] tablet 77ae087802144c45924c1b79f561afe8: '84987ad9e4a54d96af8569e4aaf0ed64' -> '7b66a8940fe54e21bec1b2db25b2fed5' move scheduled
I20240502 01:59:08.484620 17301 rebalancer_tool.cc:1074] tablet 1eb15147ff6c4d0b83b12dec736fc576: '3b8d615c71e44ff887e0f037c3efcff7' -> '26a3d5ced97d4dcc919b80fde53f6a5b' move scheduled
I20240502 01:59:08.637146 17301 rebalancer_tool.cc:1136] tablet 30620f4774b8414eb4e27bd5ecb6ecee: '7b66a8940fe54e21bec1b2db25b2fed5' -> '3b8d615c71e44ff887e0f037c3efcff7' move completed: Incomplete: tablet 30620f4774b8414eb4e27bd5ecb6ecee, TS 7b66a8940fe54e21bec1b2db25b2fed5 -> TS 3b8d615c71e44ff887e0f037c3efcff7 move failed, target replica disappeared
I20240502 01:59:08.672500 17301 rebalancer_tool.cc:1136] tablet 490e37fac0844a08a07a4b281bc33bf9: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'd4e5422f66454511ae5a50ec6ab134ad' move completed: OK
I20240502 01:59:08.685176 17301 rebalancer_tool.cc:1136] tablet ac5ec2c673cb4b988d1099e0442ec63f: 'f77c2a969bde45b49ac1bbbe242ff171' -> '84987ad9e4a54d96af8569e4aaf0ed64' move completed: Incomplete: tablet ac5ec2c673cb4b988d1099e0442ec63f, TS f77c2a969bde45b49ac1bbbe242ff171 -> TS 84987ad9e4a54d96af8569e4aaf0ed64 move failed, target replica disappeared
I20240502 01:59:08.889190 17301 rebalancer_tool.cc:1136] tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'd4e5422f66454511ae5a50ec6ab134ad' move completed: Incomplete: tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532, TS 26a3d5ced97d4dcc919b80fde53f6a5b -> TS d4e5422f66454511ae5a50ec6ab134ad move failed, target replica disappeared
I20240502 01:59:08.900961 17301 rebalancer_tool.cc:1136] tablet f97c5c9bb4c84957857efc76d1c7717f: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '3b8d615c71e44ff887e0f037c3efcff7' move completed: Incomplete: tablet f97c5c9bb4c84957857efc76d1c7717f, TS 26a3d5ced97d4dcc919b80fde53f6a5b -> TS 3b8d615c71e44ff887e0f037c3efcff7 move failed, target replica disappeared
I20240502 01:59:08.914551 17301 rebalancer_tool.cc:1136] tablet 6db29f2516e1486782be0312c9bcac8b: 'ddae2a636423489693c12b7d7d166c09' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move completed: OK
I20240502 01:59:08.922674 17301 rebalancer_tool.cc:1136] tablet cd00409d47954ee2b7555e7ff09a6e28: 'ddae2a636423489693c12b7d7d166c09' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move completed: OK
I20240502 01:59:08.941677 17301 rebalancer_tool.cc:1136] tablet e00d1ed5cde044cb9bdebf787590261a: 'f77c2a969bde45b49ac1bbbe242ff171' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move completed: OK
I20240502 01:59:09.053297 17301 rebalancer_tool.cc:1136] tablet 091d27d51acc4db0a1c7d20deeff5c10: '7b66a8940fe54e21bec1b2db25b2fed5' -> '84987ad9e4a54d96af8569e4aaf0ed64' move completed: Incomplete: tablet 091d27d51acc4db0a1c7d20deeff5c10, TS 7b66a8940fe54e21bec1b2db25b2fed5 -> TS 84987ad9e4a54d96af8569e4aaf0ed64 move failed, target replica disappeared
I20240502 01:59:09.233935 17301 rebalancer_tool.cc:1074] tablet 21e2e55d1a0d4b6eb18b4ade9468607f: 'd4e5422f66454511ae5a50ec6ab134ad' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:09.264602 17301 rebalancer_tool.cc:1074] tablet d24986c2ebab46f086e3f1f5a53af26b: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '7b66a8940fe54e21bec1b2db25b2fed5' move scheduled
I20240502 01:59:09.287379 17301 rebalancer_tool.cc:1074] tablet ce85785e670a4264b44797bb27cb4ecb: '3b8d615c71e44ff887e0f037c3efcff7' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:10.842499 17301 rebalancer_tool.cc:1136] tablet 21e2e55d1a0d4b6eb18b4ade9468607f: 'd4e5422f66454511ae5a50ec6ab134ad' -> '3b8d615c71e44ff887e0f037c3efcff7' move completed: Incomplete: tablet 21e2e55d1a0d4b6eb18b4ade9468607f, TS d4e5422f66454511ae5a50ec6ab134ad -> TS 3b8d615c71e44ff887e0f037c3efcff7 move failed, target replica disappeared
I20240502 01:59:10.911911 17301 rebalancer_tool.cc:1084] tablet a901eb3063ea4911a822552db01d365d: '3b8d615c71e44ff887e0f037c3efcff7' -> '7b66a8940fe54e21bec1b2db25b2fed5' move ignored: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }.
Pending config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: true } }
I20240502 01:59:10.912001 17301 rebalancer_tool.cc:756] re-synchronizing cluster state
Illegal state: tablet server d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): unacceptable health status UNAVAILABLE
I20240502 01:59:11.078876 18767 raft_consensus.cc:1050] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
W20240502 01:59:11.078974 18767 raft_consensus.cc:1053] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: Unable to promote non-voter 84987ad9e4a54d96af8569e4aaf0ed64: Service unavailable: leader transfer in progress
I20240502 01:59:11.110447 18416 raft_consensus.cc:1050] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171: attempting to promote NON_VOTER 7b66a8940fe54e21bec1b2db25b2fed5 to VOTER
I20240502 01:59:11.110909 18416 consensus_queue.cc:237] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } }
I20240502 01:59:11.112195 15197 raft_consensus.cc:1261] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 01:59:11.112308 15464 raft_consensus.cc:1261] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEARNER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 01:59:11.112809 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 01:59:11.112900 20213 consensus_queue.cc:1035] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:11.113062 18416 consensus_queue.cc:1035] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 01:59:11.114687 18416 raft_consensus.cc:2932] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } }
I20240502 01:59:11.115116 15464 raft_consensus.cc:2932] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } }
I20240502 01:59:11.116976 15197 raft_consensus.cc:2932] T 6db29f2516e1486782be0312c9bcac8b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } }
I20240502 01:59:11.118036 14910 catalog_manager.cc:5484] T 6db29f2516e1486782be0312c9bcac8b P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 7 to 8, 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } } }
W20240502 01:59:11.158779 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:11.159500 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:11.247157 14901 catalog_manager.cc:4573] TS 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323): ChangeConfig:ADD_PEER:NON_VOTER RPC failed for tablet 3f938c5af51a43c9a23b753c4991e41d: Remote error: Service unavailable: leader transfer in progress
I20240502 01:59:11.257884 20955 raft_consensus.cc:979] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b: : Instructing follower 84987ad9e4a54d96af8569e4aaf0ed64 to start an election
I20240502 01:59:11.257989 20955 raft_consensus.cc:1067] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Signalling peer 84987ad9e4a54d96af8569e4aaf0ed64 to start an election
I20240502 01:59:11.258392 16838 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "d24986c2ebab46f086e3f1f5a53af26b"
dest_uuid: "84987ad9e4a54d96af8569e4aaf0ed64"
from {username='slave'} at 127.7.221.66:49757
I20240502 01:59:11.258520 16838 raft_consensus.cc:482] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 01:59:11.258622 16838 raft_consensus.cc:3037] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:59:11.259441 16838 raft_consensus.cc:504] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } }
I20240502 01:59:11.259958 16838 leader_election.cc:288] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [CANDIDATE]: Term 2 election: Requested vote from peers 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
I20240502 01:59:11.260246 15197 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d24986c2ebab46f086e3f1f5a53af26b" candidate_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" candidate_term: 2 candidate_status { last_received { term: 1 index: 12 } } ignore_live_leader: true dest_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b"
I20240502 01:59:11.260385 15197 raft_consensus.cc:3032] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Stepping down as leader of term 1
I20240502 01:59:11.260430 15197 raft_consensus.cc:729] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Running, Role: LEADER
I20240502 01:59:11.260569 15197 consensus_queue.cc:260] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 12, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } }
I20240502 01:59:11.260637 15464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d24986c2ebab46f086e3f1f5a53af26b" candidate_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" candidate_term: 2 candidate_status { last_received { term: 1 index: 12 } } ignore_live_leader: true dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5"
I20240502 01:59:11.260758 15464 raft_consensus.cc:3037] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Advancing to term 2
I20240502 01:59:11.260787 15197 raft_consensus.cc:3037] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Advancing to term 2
W20240502 01:59:11.261085 16771 leader_election.cc:334] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:11.261765 15197 raft_consensus.cc:2454] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84987ad9e4a54d96af8569e4aaf0ed64 in term 2.
I20240502 01:59:11.261768 15464 raft_consensus.cc:2454] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84987ad9e4a54d96af8569e4aaf0ed64 in term 2.
I20240502 01:59:11.262161 16772 leader_election.cc:302] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 4 voters: 3 yes votes; 1 no votes. yes voters: 26a3d5ced97d4dcc919b80fde53f6a5b, 7b66a8940fe54e21bec1b2db25b2fed5, 84987ad9e4a54d96af8569e4aaf0ed64; no voters: fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:11.262341 20110 raft_consensus.cc:2781] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 01:59:11.262495 20110 raft_consensus.cc:686] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [term 2 LEADER]: Becoming Leader. State: Replica: 84987ad9e4a54d96af8569e4aaf0ed64, State: Running, Role: LEADER
I20240502 01:59:11.262670 20110 consensus_queue.cc:237] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 12, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } }
W20240502 01:59:11.263916 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 01:59:11.264432 14913 catalog_manager.cc:5484] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 reported cstate change: term changed from 1 to 2, leader changed from 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) to 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69). New cstate: current_term: 2 leader_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: NON_VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
W20240502 01:59:11.266319 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:11.270076 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:11.281477 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:11.295154 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:11.381139 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:11.444969 20955 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:11.446421 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:11.456627 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:11.460412 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:11.506902 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:11.611987 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:11.633678 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:11.683450 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:11.691476 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 01:59:11.695911 18767 raft_consensus.cc:1050] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
W20240502 01:59:11.695989 18767 raft_consensus.cc:1053] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: Unable to promote non-voter 84987ad9e4a54d96af8569e4aaf0ed64: Service unavailable: leader transfer in progress
W20240502 01:59:11.710196 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:11.723497 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 01:59:11.748908 18767 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:11.749044 18767 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:11.749541 18767 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 01:59:11.750042 15328 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:59:11.750180 15328 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 01:59:11.751247 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:11.751329 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:11.751374 15396 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:11.751732 18767 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
I20240502 01:59:11.752749 15197 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P 26a3d5ced97d4dcc919b80fde53f6a5b [term 2 FOLLOWER]: Refusing update from remote peer 84987ad9e4a54d96af8569e4aaf0ed64: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 2 index: 13. (index mismatch)
I20240502 01:59:11.753126 20110 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
W20240502 01:59:11.770908 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 01:59:11.779078 15464 raft_consensus.cc:1261] T d24986c2ebab46f086e3f1f5a53af26b P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Refusing update from remote peer 84987ad9e4a54d96af8569e4aaf0ed64: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 2 index: 13. (index mismatch)
I20240502 01:59:11.779475 20110 consensus_queue.cc:1035] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
W20240502 01:59:11.783135 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:11.814610 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:11.883915 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:11.903576 18109 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:12.035243 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:12.036530 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:12.081682 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:12.083452 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:12.089255 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:12.091190 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:12.114938 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:12.197319 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 01:59:12.200685 18767 raft_consensus.cc:1050] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5: attempting to promote NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 to VOTER
I20240502 01:59:12.201079 18767 consensus_queue.cc:237] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 15, Committed index: 15, Last appended: 1.15, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 01:59:12.202304 16838 raft_consensus.cc:1261] T 3f938c5af51a43c9a23b753c4991e41d P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 LEARNER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 01:59:12.202751 21464 consensus_queue.cc:1035] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
W20240502 01:59:12.203122 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:12.203122 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 01:59:12.245558 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:12.245719 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:12.248164 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:12.297210 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:12.305142 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:12.306658 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:12.329613 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:12.376853 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 01:59:12.388258 20955 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:12.429831 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:12.460280 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:12.466691 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:12.506976 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:12.790660 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 01:59:12.878708 20955 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:12.884469 14902 catalog_manager.cc:4573] TS 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 21e2e55d1a0d4b6eb18b4ade9468607f: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:12.898226 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:12.981953 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:13.104359 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:13.313637 14901 catalog_manager.cc:5014] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 3f938c5af51a43c9a23b753c4991e41d with cas_config_opid_index 15: ChangeConfig:ADD_PEER:NON_VOTER failed with leader 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323) due to error UNKNOWN_ERROR; will retry: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }.
Pending config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
W20240502 01:59:13.360906 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 01:59:13.488237 21727 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:13.503845 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:13.564941 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:13.577755 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:13.598804 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:13.599862 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:13.691665 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:13.691723 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:13.702919 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:13.746459 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 01:59:13.762879 21466 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:13.763013 21466 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:13.763566 21466 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 01:59:13.763736 15328 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:59:13.763881 15328 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 01:59:13.764861 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:13.765020 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:13.765100 15396 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:13.765233 21466 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 01:59:13.821709 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:13.865643 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:13.930183 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.033434 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 105 similar messages]
W20240502 01:59:14.034101 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:14.046211 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 93 similar messages]
I20240502 01:59:14.066215 21727 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:14.070720 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 89 similar messages]
W20240502 01:59:14.071403 15266 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 98 similar messages]
W20240502 01:59:14.071938 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.087739 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.109198 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.159497 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 01:59:14.160159 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:14.248994 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:14.272945 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.275624 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:14.307332 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.397876 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.399260 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.449705 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.483027 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:14.525357 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.556030 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:14.620922 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:14.651845 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 01:59:14.667708 21727 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:14.682612 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:14.695878 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.703097 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:14.728941 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.758991 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 01:59:14.792982 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.844621 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.875555 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.883667 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.884660 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.896919 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.898021 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.918890 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:14.952374 14901 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 01:59:14.953071 14901 catalog_manager.cc:4573] TS d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 21e2e55d1a0d4b6eb18b4ade9468607f: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:15.024614 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:15.026383 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:15.038586 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:15.067423 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 01:59:15.121336 21727 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:15.209234 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:15.285017 21726 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:15.285122 21726 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:15.285670 21726 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 01:59:15.285960 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:59:15.286113 15465 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 01:59:15.286514 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:15.287307 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:15.287744 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:15.287818 15263 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:15.287945 21726 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 01:59:15.423844 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:15.530680 21727 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:15.689057 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:15.808338 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:15.826786 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:15.913918 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:15.947117 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:15.953934 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:16.035341 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:16.074802 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:16.109388 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 01:59:16.150126 21727 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:16.226524 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:16.239912 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:16.253960 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:16.269207 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:16.418818 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:16.431380 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:16.451112 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:16.599702 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:16.648002 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:16.682446 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 01:59:16.690936 22302 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:16.729007 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:16.749114 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:16.768229 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:16.773028 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:16.851119 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:16.973737 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.001894 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.076269 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:17.092572 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.120203 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.132004 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:17.170126 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:17.182026 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 01:59:17.194787 22516 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:17.196228 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.197322 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.214864 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:17.224807 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 01:59:17.259146 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.276786 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.339831 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.419672 14901 catalog_manager.cc:5014] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 3f938c5af51a43c9a23b753c4991e41d with cas_config_opid_index 15: ChangeConfig:ADD_PEER:NON_VOTER failed with leader 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323) due to error UNKNOWN_ERROR; will retry: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }.
Pending config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
W20240502 01:59:17.428346 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.453521 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.485265 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.556915 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.558419 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:17.566644 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 01:59:17.605062 22516 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:17.607750 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:17.626080 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.627966 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.800426 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:17.837186 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:17.874357 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:18.013041 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:18.109272 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:18.114125 22725 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 01:59:18.221398 22462 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:18.221524 22462 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:18.221999 22462 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 01:59:18.222328 15464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:59:18.222492 15464 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 01:59:18.223196 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:18.223443 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:18.223507 15263 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:18.223618 22462 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 01:59:18.430048 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:18.491637 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:18.492831 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 01:59:18.503706 22516 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:18.516767 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:18.557664 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:18.577996 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:18.700241 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:18.704942 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:18.726298 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:18.731673 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:18.797048 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:18.832641 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:18.856617 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:18.977222 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 01:59:19.043262 22882 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:19.066013 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:19.069937 14901 catalog_manager.cc:4573] TS d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 21e2e55d1a0d4b6eb18b4ade9468607f: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:19.071986 14994 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 98 similar messages]
W20240502 01:59:19.077355 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 01:59:19.078137 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:19.100281 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 71 similar messages]
W20240502 01:59:19.141623 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 96 similar messages]
W20240502 01:59:19.172210 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:19.190904 16773 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 48 similar messages]
W20240502 01:59:19.231972 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:19.272790 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:19.323745 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:19.336272 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:19.397395 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:19.421049 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:19.421425 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:19.506515 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:19.526108 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:19.565202 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 01:59:19.565932 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:19.567912 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:19.600248 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:19.607477 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:19.623099 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:19.723150 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:19.763620 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:19.780448 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:19.803848 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 01:59:19.850266 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:19.865089 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:19.884071 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:19.941542 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:19.974833 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:20.055670 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:20.057631 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:20.122809 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:20.141202 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 01:59:20.146628 22620 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:20.146762 22620 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:20.147280 22620 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 01:59:20.148125 15328 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:59:20.148260 15328 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 01:59:20.148375 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:20.148579 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:20.148655 15396 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:20.148792 22620 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
I20240502 01:59:20.159233 22882 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:20.169359 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:20.306324 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:20.414048 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:20.448760 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:20.484476 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 01:59:20.561003 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:20.630261 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:20.729769 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:20.753731 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:20.977178 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:21.041641 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:21.079254 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:21.083353 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:21.097605 23306 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:21.105309 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:21.108714 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:21.130898 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:21.232450 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:21.260916 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:21.299404 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:21.529440 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:21.549384 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:21.552177 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 01:59:21.557276 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:21.594373 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:21.595217 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:21.620819 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:21.629345 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:21.634054 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:21.671299 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:21.701900 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:21.738693 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:21.937870 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:21.946257 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:21.991190 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:21.997947 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:22.001726 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.017127 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.042066 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:22.121637 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.149857 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.242867 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.253384 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:22.328115 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.351871 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.355710 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 01:59:22.371507 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.379133 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.394312 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:22.401827 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:22.411293 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:22.468380 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.468477 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.503762 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.543304 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.562671 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.713860 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.725462 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:22.781757 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:22.808148 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:22.882609 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:22.966940 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:23.080204 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:23.138701 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:23.258878 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:23.348932 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:23.382458 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:23.417032 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 01:59:23.423839 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:23.460347 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:23.664343 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:23.717096 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:23.757148 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:23.787994 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:23.796923 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:23.848516 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:23.872123 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:23.890966 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:23.897308 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:23.941123 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 01:59:24.006690 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:24.034166 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.070539 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.077643 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.106470 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 01:59:24.107111 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.111923 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 83 similar messages]
W20240502 01:59:24.139196 14992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 101 similar messages]
W20240502 01:59:24.212509 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 49 similar messages]
W20240502 01:59:24.213205 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:24.256675 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 89 similar messages]
W20240502 01:59:24.257302 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:24.308743 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:24.324796 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.406984 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.410579 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 01:59:24.495630 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:24.520258 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:24.663376 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.692792 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.697724 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.705013 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:24.727655 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:24.783697 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.789631 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.802407 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.810166 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.856165 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.862367 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.862367 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 01:59:24.904218 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:24.933130 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:24.940699 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 01:59:24.955744 23930 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:24.955873 23930 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:24.956370 23930 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 01:59:24.956590 15328 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:59:24.956737 15328 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 01:59:24.957608 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:24.958164 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:24.958240 15399 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:24.958473 23930 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
I20240502 01:59:25.054770 24138 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:25.057438 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:25.106864 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:25.164958 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:25.179211 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:25.205236 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:25.422034 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:25.477198 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 01:59:25.509228 24138 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:25.540503 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:25.575361 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:25.619189 14901 catalog_manager.cc:5014] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 3f938c5af51a43c9a23b753c4991e41d with cas_config_opid_index 15: ChangeConfig:ADD_PEER:NON_VOTER failed with leader 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323) due to error UNKNOWN_ERROR; will retry: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }.
Pending config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
W20240502 01:59:25.673218 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:25.730871 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:25.852603 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:25.859799 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 01:59:25.886456 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:25.994586 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:26.069411 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:26.079478 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:26.161532 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:26.184437 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:26.248257 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:26.248924 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:26.294245 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 01:59:26.385799 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:26.427482 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:26.453162 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:26.457518 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:26.464633 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:26.484088 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:26.553661 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:26.667475 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:26.745074 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:26.767359 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:26.767987 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 01:59:26.877998 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:26.897001 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:26.980124 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:27.043699 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:27.061790 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:27.072865 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:27.203673 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:27.233340 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:27.243661 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:27.247954 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:27.259815 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:27.269578 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 01:59:27.273913 14901 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 5 similar messages]
W20240502 01:59:27.274617 14901 catalog_manager.cc:4573] TS d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 21e2e55d1a0d4b6eb18b4ade9468607f: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:27.288919 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:27.316082 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:27.365598 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:27.366194 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:27.385057 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:27.389782 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:27.396458 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:27.408402 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:27.424109 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 01:59:27.443746 24608 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:27.623850 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:27.673851 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:27.684751 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:27.690479 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:27.803805 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:27.994467 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:27.997386 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:28.051496 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:28.127843 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:28.194530 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:28.233016 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:28.267941 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:28.270789 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:28.358378 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:28.373514 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:28.446919 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 01:59:28.550559 24608 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:28.577255 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:28.608703 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:28.636449 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:28.654964 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:28.655604 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:28.723426 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:28.740281 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:28.915261 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:29.013391 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:29.032423 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:29.064462 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:29.068754 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:29.143362 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 99 similar messages]
W20240502 01:59:29.144223 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:29.147820 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:29.165374 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 68 similar messages]
I20240502 01:59:29.175611 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:29.209236 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:29.259763 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 83 similar messages]
W20240502 01:59:29.302251 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 01:59:29.312402 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:29.423730 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 51 similar messages]
W20240502 01:59:29.501341 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:29.542980 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:29.565490 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:29.649410 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:29.670706 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:29.688865 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:29.786674 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:29.792469 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:29.793573 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 01:59:29.799168 25132 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:29.816556 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 01:59:29.838908 24721 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:29.839042 24721 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:29.839532 24721 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 01:59:29.839915 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:59:29.840065 15465 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 01:59:29.840670 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:29.840852 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:29.840929 15263 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:29.841058 24721 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 01:59:29.859927 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:29.915380 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:29.916208 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:29.961135 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 01:59:29.984779 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:30.037292 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:30.050431 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:30.052268 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:30.107657 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:30.125586 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:30.127172 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:30.132113 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:30.190969 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:30.200969 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:30.306687 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:30.308669 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
I20240502 01:59:30.369052 25132 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:30.435012 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:30.568428 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:30.600097 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:30.616029 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:30.656340 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:30.710017 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:30.766650 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
I20240502 01:59:30.813679 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:30.957825 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:31.012545 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:31.064034 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:31.064743 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:31.078764 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:31.106887 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:31.166179 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
I20240502 01:59:31.238723 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:31.243562 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:31.248662 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:31.342144 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:31.427613 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:31.432601 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:31.483758 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:31.503355 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:31.516222 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:31.517357 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:31.774355 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
I20240502 01:59:31.792496 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:31.803238 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:31.868765 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:31.982622 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:31.994939 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:31.997851 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:32.016139 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:32.157764 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:32.190702 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
I20240502 01:59:32.193369 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:32.204627 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:32.273461 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:32.400147 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:32.414754 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 01:59:32.421669 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:32.431252 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:32.451054 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:32.463970 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:32.464123 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:32.486428 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:32.504566 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:32.534441 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:32.559253 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:32.594888 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 01:59:32.700915 25702 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:32.807785 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:32.830866 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:32.845314 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:32.865561 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:32.869625 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:32.901145 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:32.999049 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:33.009536 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:33.039290 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:33.101308 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:33.246048 25702 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:33.287460 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:33.295506 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:33.322299 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:33.394122 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:33.427898 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:33.467242 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:33.592713 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:33.625090 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
I20240502 01:59:33.627898 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:33.650902 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:33.715426 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:33.739018 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:33.765796 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:33.813999 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:33.875494 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:33.940558 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:33.969738 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:34.111482 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:34.169445 25702 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:34.177238 14994 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 100 similar messages]
W20240502 01:59:34.210038 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 70 similar messages]
W20240502 01:59:34.210645 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:34.254483 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:34.256161 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:34.272464 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 01:59:34.339262 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 01:59:34.373767 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:34.424901 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 47 similar messages]
W20240502 01:59:34.425568 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:34.458801 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:34.545661 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:34.554464 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
I20240502 01:59:34.565961 25702 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:34.573824 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:34.593896 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:34.656613 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:34.720466 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:34.745811 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:34.765545 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:34.800796 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:34.801280 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:34.802795 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
I20240502 01:59:34.912551 25967 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:34.912688 25967 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:34.913142 25967 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 01:59:34.913375 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:59:34.913511 15465 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 01:59:34.914067 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:34.914153 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:34.914222 15263 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:34.914333 25967 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 01:59:34.925897 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
I20240502 01:59:35.006291 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:35.043931 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:35.091655 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:35.102699 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:35.179390 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:35.183205 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:35.189354 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 01:59:35.265807 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:35.333082 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:35.334535 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:35.344470 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:35.364961 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:35.408830 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:35.464193 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
I20240502 01:59:35.482466 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:35.556406 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:35.567663 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:35.621403 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:35.654071 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:35.847138 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:35.868883 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:35.920773 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:35.994535 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:36.003069 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
I20240502 01:59:36.024713 22991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:36.034579 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:36.071028 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:36.138455 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:36.287660 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:36.290023 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:36.291985 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:36.317230 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:36.371135 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:36.403800 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
I20240502 01:59:36.410989 26099 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:36.454068 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:36.475601 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:36.651304 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:36.740471 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:36.798763 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:36.870064 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:36.945096 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:36.947347 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:36.985607 26592 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:37.027820 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.064283 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.117113 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:37.168126 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.281730 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.287284 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.344367 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.355598 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:37.389117 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:37.418315 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
I20240502 01:59:37.422932 26592 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:37.438501 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.449957 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:37.451584 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.547530 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.561534 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.570276 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.595504 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:37.778472 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.806001 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.828191 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:37.934895 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.937211 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 01:59:37.939358 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:37.945582 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.947679 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:37.990011 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:38.008342 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:38.032049 26099 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:38.050618 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
I20240502 01:59:38.119014 26801 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:38.119138 26801 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:38.119625 26801 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 01:59:38.119890 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:59:38.120050 15465 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 01:59:38.120807 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:38.120973 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:38.121026 15266 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:38.121160 26801 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 01:59:38.191673 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:38.274765 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:38.313082 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:38.390305 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:38.427366 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:38.477519 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:38.496234 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:38.511677 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:38.554590 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
I20240502 01:59:38.558018 26908 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:38.782958 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:38.805312 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:38.848980 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:38.882386 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:38.946214 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:38.957345 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:38.978771 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:39.005533 26099 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:39.019034 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:39.085875 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:39.222193 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 100 similar messages]
W20240502 01:59:39.242851 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:39.288033 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:39.294618 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 75 similar messages]
W20240502 01:59:39.328105 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 01:59:39.379557 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:39.399740 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:39.414186 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 94 similar messages]
W20240502 01:59:39.483443 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 01:59:39.483556 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 01:59:39.505281 26099 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:39.512218 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:39.585152 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:39.588521 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:39.635198 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:39.675117 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:39.751770 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:39.761807 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:39.793551 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:39.795464 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:39.870882 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 01:59:39.884114 26099 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:39.922051 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:39.938990 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:39.994195 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:40.021138 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:40.076944 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:40.136953 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:40.210933 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:40.262624 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:40.310954 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:40.320426 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:40.383126 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:40.383631 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:40.384187 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:40.402004 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:40.447407 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 01:59:40.461705 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
I20240502 01:59:40.483335 26099 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:40.614550 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:40.619989 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:40.629120 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:40.778219 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:40.780409 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:40.801834 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:40.828563 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:40.955552 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
I20240502 01:59:40.963568 26908 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:41.073143 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:41.191798 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:41.226455 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:41.230031 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:41.275379 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:41.354384 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:41.370155 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:41.412096 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:41.513896 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:41.516041 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:41.533071 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
I20240502 01:59:41.534977 26908 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:41.601498 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:41.657946 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:41.723799 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:41.782550 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:41.804246 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:41.839221 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:41.841550 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:41.874018 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:41.978328 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
I20240502 01:59:42.010839 27533 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:42.018093 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:42.027730 14901 catalog_manager.cc:5014] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 3f938c5af51a43c9a23b753c4991e41d with cas_config_opid_index 15: ChangeConfig:ADD_PEER:NON_VOTER failed with leader 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323) due to error UNKNOWN_ERROR; will retry: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }.
Pending config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
W20240502 01:59:42.070940 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:42.088872 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:42.217625 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:42.223582 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:42.238885 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:42.257171 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:42.311201 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:42.361434 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:42.395088 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
I20240502 01:59:42.428242 26908 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:42.587592 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:42.607290 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 01:59:42.626600 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:42.681737 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:42.711946 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:42.714653 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:42.827494 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:42.831606 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:42.846617 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
I20240502 01:59:42.850793 27533 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:42.872784 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:42.882066 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:42.942975 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:42.977186 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:43.090198 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:43.116853 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:43.117954 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:43.118257 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:43.206228 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:43.208640 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:43.366962 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
I20240502 01:59:43.412271 27533 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:43.417130 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:43.550606 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:43.639798 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:43.681293 14901 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 2 similar messages]
W20240502 01:59:43.682044 14901 catalog_manager.cc:4573] TS d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 21e2e55d1a0d4b6eb18b4ade9468607f: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:43.722174 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:43.725173 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:43.789563 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:43.790287 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:43.838711 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:43.851549 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
I20240502 01:59:43.876008 26908 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:43.882761 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:44.040557 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:44.042702 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:44.156410 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:44.274538 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 99 similar messages]
W20240502 01:59:44.283275 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:44.289377 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:44.293414 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:44.375322 15398 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 83 similar messages]
I20240502 01:59:44.384506 28104 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:44.390329 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:44.403463 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:44.411792 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:44.420642 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:44.438263 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 90 similar messages]
W20240502 01:59:44.457808 15127 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 01:59:44.502578 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 52 similar messages]
W20240502 01:59:44.540616 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:44.550129 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:44.558089 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:44.639675 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:44.641512 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:44.767469 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:44.777896 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:44.790845 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:44.809139 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
I20240502 01:59:44.858469 26908 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:44.898900 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
I20240502 01:59:44.976294 27534 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:44.976405 27534 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:44.976821 27534 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 01:59:44.977113 15464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 01:59:44.977250 15464 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 01:59:44.977752 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:44.977861 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:44.977941 15266 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:44.978085 27534 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 01:59:44.987541 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:44.994655 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 01:59:45.001338 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:45.060525 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:45.114789 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:45.175575 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:45.339654 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:45.343232 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:45.358537 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:45.366125 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:45.366582 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:45.390069 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:45.390800 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:45.392825 26908 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:45.461040 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:45.487921 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:45.506064 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:45.610836 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:45.753041 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:45.753041 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:45.785730 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:45.794632 26908 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:45.810953 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:45.889082 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:45.943194 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:45.976011 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:46.107236 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:46.219694 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:46.222283 28314 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:46.275444 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:46.289458 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:46.306178 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:46.334136 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:46.383781 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:46.406494 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:46.570050 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:46.622350 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:46.697685 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:46.715907 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:46.737246 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:46.758690 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:46.797127 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:46.812062 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
I20240502 01:59:46.827469 28314 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:46.868602 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:46.941531 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:47.069984 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:47.140834 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:47.146157 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:47.160305 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:47.263314 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:47.287477 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 01:59:47.291568 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:47.295571 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:47.299778 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:47.316138 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:47.320118 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:47.428833 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
I20240502 01:59:47.446141 28775 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:47.454195 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:47.580677 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:47.597115 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:47.599763 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:47.727162 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:47.732823 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:47.738238 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:47.755192 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:47.833763 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:47.972386 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:47.997746 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:48.046985 28888 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:48.053736 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:48.108232 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:48.123304 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:48.131734 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:48.136464 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:48.188120 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:48.242904 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:48.294764 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:48.300652 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:48.439173 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:48.492986 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:48.496954 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:48.554193 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
I20240502 01:59:48.650645 28888 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:48.839454 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:48.903883 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:48.919011 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:49.006191 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:49.055650 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.102560 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:49.103360 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.164047 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
I20240502 01:59:49.221278 29145 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:49.236923 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.323284 14992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 101 similar messages]
W20240502 01:59:49.328040 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.382913 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.400333 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:49.407213 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.414084 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 78 similar messages]
W20240502 01:59:49.417343 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:49.445118 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.497582 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 96 similar messages]
W20240502 01:59:49.527496 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 01:59:49.529234 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.544624 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.548048 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.549013 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 49 similar messages]
W20240502 01:59:49.617031 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.617283 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:49.696566 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.762584 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.768662 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
I20240502 01:59:49.804061 29246 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:49.815660 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.832212 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.841624 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:49.885216 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:49.939220 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:50.000946 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:50.026078 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:50.058229 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:50.086198 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:50.117308 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 01:59:50.202534 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:50.295217 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:50.321429 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
I20240502 01:59:50.366742 29145 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:50.392746 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:50.450126 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:50.530844 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:50.537113 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:50.578207 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:50.619184 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:50.659576 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:50.700320 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:50.829460 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:50.838543 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:50.865090 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:50.889500 29450 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:50.942802 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:51.079984 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:51.123703 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:51.137365 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:51.190167 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:51.281424 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:51.298985 29450 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:51.435050 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:51.478183 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:51.545697 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:51.602442 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:51.668537 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:51.674747 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:51.700909 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:51.772441 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:51.784649 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:51.850090 29450 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:51.958132 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:51.991734 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:52.004679 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:52.005030 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:52.081815 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:52.104080 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:52.141489 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:52.146761 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:52.177703 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:52.193893 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:52.201753 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:52.223407 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:52.240594 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
I20240502 01:59:52.242483 29450 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:52.263414 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:52.318378 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:52.324095 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:52.418076 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:52.592748 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:52.594748 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:52.597905 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:52.625371 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
I20240502 01:59:52.679796 28970 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 01:59:52.679925 28970 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 01:59:52.680435 28970 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 01:59:52.680728 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 01:59:52.680873 15329 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 01:59:52.681381 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:52.681522 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:52.681582 15399 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:52.681725 28970 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 01:59:52.698854 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
I20240502 01:59:52.774124 29450 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:52.867055 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 01:59:52.890431 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:52.895335 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:52.903590 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:52.927894 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:52.998363 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:53.100898 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:53.134871 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:53.185894 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:53.205885 29450 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:53.219045 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:53.277140 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:53.316287 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:53.330142 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:53.342880 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:53.436218 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:53.456076 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:53.551079 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:53.629889 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:53.633769 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:53.788769 30067 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:53.913445 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:53.958031 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:54.007665 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:54.047845 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:54.131976 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:54.161448 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:54.186480 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:54.309644 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:54.361083 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:54.371362 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
I20240502 01:59:54.380686 30170 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:54.427410 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:54.431133 14992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 101 similar messages]
W20240502 01:59:54.431761 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:54.440544 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 01:59:54.445799 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:54.484802 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:54.496376 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:54.506110 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:54.558198 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 89 similar messages]
W20240502 01:59:54.569134 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:54.574271 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:54.579490 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 71 similar messages]
W20240502 01:59:54.612540 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:54.613615 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 01:59:54.633846 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:54.635376 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:54.720675 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:54.784164 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
I20240502 01:59:54.850658 30067 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:54.903205 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:54.981914 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:55.017397 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:55.027559 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:55.056062 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:55.101698 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:55.184006 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:55.214154 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:55.255798 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:55.284488 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 01:59:55.373804 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:55.412340 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
I20240502 01:59:55.445255 30377 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:55.459771 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:55.512647 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:55.529647 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:55.573316 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:55.624984 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:55.648909 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:55.709815 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:55.841279 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:55.871546 30067 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:55.908696 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:55.937351 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:56.041584 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:56.042258 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:56.044703 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:56.066696 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:56.183652 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:56.324926 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 01:59:56.389814 30067 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:56.439569 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:56.515244 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:56.525619 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:56.568516 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:56.580837 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:56.609089 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:56.716975 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:56.826225 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:56.851913 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:56.894459 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
I20240502 01:59:56.897729 30690 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:56.920486 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:56.945103 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:56.960227 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:56.971505 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:56.983901 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:57.041734 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:57.047580 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:57.072204 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:57.094977 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:57.115731 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:57.169245 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:57.201215 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:57.212373 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
I20240502 01:59:57.323499 30067 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:57.326704 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:57.375053 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:57.399560 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:57.475559 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:57.569590 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:57.664184 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
I20240502 01:59:57.698781 30690 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:57.710243 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:57.746555 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:57.785748 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 01:59:57.861148 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:57.900868 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:57.980386 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:57.982007 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:57.985020 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:58.039389 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:58.053661 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
I20240502 01:59:58.089946 30067 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:58.123631 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:58.278398 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:58.304875 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:58.365291 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:58.399699 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:58.406625 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:58.408082 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:58.427523 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
I20240502 01:59:58.530113 30690 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:58.668921 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:58.715015 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:58.777459 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:58.803814 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:58.805162 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:58.840258 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:58.923255 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 01:59:58.980880 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
I20240502 01:59:58.994941 30067 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:59.082226 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:59.156221 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:59.288283 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:59.362843 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:59.373406 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:59.395823 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:59.410959 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:59.456601 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 01:59:59.457188 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:59.482978 30690 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 01:59:59.513427 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:59.537254 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:59.545539 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:59.549703 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 101 similar messages]
W20240502 01:59:59.550004 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 01:59:59.550393 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:59.583317 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 01:59:59.583920 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:59.613448 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:59.613443 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:59.655356 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 01:59:59.655936 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:59.676056 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:59.677349 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 01:59:59.801896 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 01:59:59.806444 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 01:59:59.890071 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 01:59:59.910004 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 01:59:59.945079 30690 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:00.078928 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 02:00:00.147468 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 02:00:00.221141 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:00.228125 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 02:00:00.360183 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 02:00:00.376058 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:00.399222 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 02:00:00.403584 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:00.423090 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 02:00:00.427139 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:00.456652 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:00:00.468159 31413 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:00.475214 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 02:00:00.527952 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:00.577384 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 02:00:00.731652 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:00.777331 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 02:00:00.821928 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:00.882480 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:01.046211 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:00:01.053969 30690 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:01.072911 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:01.213151 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 02:00:01.249884 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:01.278093 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
I20240502 02:00:01.278086 31211 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:00:01.278219 31211 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:00:01.278653 31211 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:00:01.279013 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:00:01.279156 15329 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:00:01.279385 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:01.279809 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:00:01.279870 15396 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:00:01.279985 31211 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:00:01.289912 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:01.343621 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 02:00:01.355305 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:01.430081 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:01.435489 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
I20240502 02:00:01.444260 30690 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:01.490140 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:01.510463 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:01.632642 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 02:00:01.780293 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:01.858603 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:01.889150 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:01.930470 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:01.964608 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 02:00:01.969763 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:02.001863 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:02.020301 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:02.035717 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
I20240502 02:00:02.036821 31729 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:02.045169 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:02.062103 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:02.086397 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:02.106225 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:02.160470 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:02.231263 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:02.278396 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:02.334553 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:02.360309 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:02.398485 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:00:02.481989 30690 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:02.592120 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:02.649466 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:02.715458 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 02:00:02.720650 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:02.759280 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:02.762413 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:02.769693 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:02.786355 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:02.788463 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:02.825744 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:02.835129 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:02.878721 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
I20240502 02:00:02.883739 30690 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:02.971496 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:03.105712 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:03.204119 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:03.262562 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:03.280647 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:03.285619 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:03.330802 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
I20240502 02:00:03.402333 30690 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:03.432119 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:03.559634 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:03.646121 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:03.653756 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:03.724117 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:03.757026 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:03.757524 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:03.761338 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:03.835080 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:03.914155 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
I20240502 02:00:03.921669 32141 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:03.970438 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:03.989059 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:04.102795 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:04.106685 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:04.207547 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:04.224494 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 02:00:04.369076 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:04.370231 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:04.374080 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:04.409533 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
I20240502 02:00:04.477749 32141 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:04.495512 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:04.544613 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:04.585124 14992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 100 similar messages]
W20240502 02:00:04.595552 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 02:00:04.597982 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 02:00:04.627017 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:04.627478 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:04.664461 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:04.671653 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:04.685144 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 71 similar messages]
W20240502 02:00:04.695355 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:04.768008 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 48 similar messages]
W20240502 02:00:04.778168 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:04.779949 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:04.794270 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
I20240502 02:00:04.859169 30690 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:04.957477 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:04.977886 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:05.019367 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:05.133709 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:05.166947 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:05.172628 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:05.183941 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:05.204100 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:05.219566 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:05.254882 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:05.348737 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
I20240502 02:00:05.365543 30690 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:05.435772 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:05.464596 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:05.476231 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:05.548908 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:05.626312 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:05.652948 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:05.658800 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:05.672282 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:00:05.742801 30690 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:05.742952 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:05.820501 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:05.895779 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:05.951443 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:06.045904 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:06.099156 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:06.134505 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:06.160130 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:06.205976 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:00:06.297557 30690 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:06.311117 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:06.410166 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:06.427690 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:06.498029 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:06.527351 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:06.596771 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:06.638159 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:06.663569 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:06.675427 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:06.688812 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:06.739955 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
I20240502 02:00:06.754945 32604 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:06.918723 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:06.922307 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:06.953078 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:06.958940 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:07.046016 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:07.148591 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:07.164754 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:07.200119 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:07.219226 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:07.256886 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:07.279628 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:07.337721 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
I20240502 02:00:07.342576 346 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:07.343299 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:07.348255 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:07.398433 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:07.406867 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:07.435372 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:07.511655 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:07.523762 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:07.587077 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:07.677332 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:07.680680 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:07.774266 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:07.855706 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
I20240502 02:00:07.891804 346 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:07.914443 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:07.935528 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 02:00:08.096570 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:08.100378 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:08.118067 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:08.164377 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:08.173727 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:08.259404 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:08.275468 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:08.352480 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:00:08.356647 346 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:08.397394 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:08.492143 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:08.662005 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:00:08.667340 31728 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:00:08.667447 31728 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:00:08.667884 31728 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:00:08.668148 15464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 02:00:08.668283 15464 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 02:00:08.668761 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:08.668915 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:00:08.668980 15266 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:00:08.669088 31728 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:00:08.696533 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:08.793481 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:08.813113 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:08.819416 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:08.821199 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:08.856225 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:08.874135 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:08.887724 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:08.900839 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
I20240502 02:00:08.907148 698 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 02:00:08.956189 15174 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 02:00:08.958621 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 02:00:08.959939 15444 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 02:00:08.962906 15309 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 02:00:08.963204 16820 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:00:09.003702 17300 rebalancer_tool-test.cc:1260] rebalancer run info: Runtime error: /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu: process exited with non-zero status 1
stdout: Per-server replica distribution summary:
Statistic | Value
-----------------------+-----------
Minimum Replica Count | 0
Maximum Replica Count | 23
Average Replica Count | 11.250000
Per-table replica distribution summary:
Replica Skew | Value
--------------+----------
Minimum | 3
Maximum | 3
Average | 3.000000
stderr: W20240502 01:59:03.142086 17303 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:59:03.142328 17303 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:59:03.146576 17303 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 01:59:04.180335 17303 rebalancer_tool.cc:215] running whole-cluster rebalancing
W20240502 01:59:04.703430 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.703565 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.703613 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 9736e7c5365849759564e386d208932f: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.703656 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:04.703720 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.703775 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.703823 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.704012 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.704068 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:04.704145 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0b29d58fe9c74d1cb4345bb6cfbd0535: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.704209 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.704277 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.704342 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.704412 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:04.742259 17303 rebalancer_tool.cc:1074] tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move scheduled
I20240502 01:59:04.779028 17303 rebalancer_tool.cc:1074] tablet 856f623374824d94913db0cb9faa25d1: 'f77c2a969bde45b49ac1bbbe242ff171' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:04.827572 17303 rebalancer_tool.cc:1074] tablet fc07e430e3e74e3885f263881bd04976: 'ddae2a636423489693c12b7d7d166c09' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:05.006157 17303 rebalancer_tool.cc:1074] tablet bf73e677bcd049869cadf9f8c9cd3434: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:05.032227 17303 rebalancer_tool.cc:1084] tablet ac5ec2c673cb4b988d1099e0442ec63f: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move ignored: Illegal state: tablet ac5ec2c673cb4b988d1099e0442ec63f: replica fc067ee690bb457e8109ad8e4eb43ab8 is already present
I20240502 01:59:05.032311 17303 rebalancer_tool.cc:756] re-synchronizing cluster state
W20240502 01:59:05.574771 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:05.574911 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:05.574960 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:05.575008 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:05.575045 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 9736e7c5365849759564e386d208932f: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.586691 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:05.586764 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.586809 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.586849 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:05.586884 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:05.586920 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0b29d58fe9c74d1cb4345bb6cfbd0535: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:05.586973 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:05.797588 17303 rebalancer_tool.cc:1074] tablet 77ae087802144c45924c1b79f561afe8: 'f77c2a969bde45b49ac1bbbe242ff171' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:05.987113 17303 rebalancer_tool.cc:1074] tablet 6c4e85e7050b4dc8a4ce6bba818854a2: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move scheduled
I20240502 01:59:06.178254 17303 rebalancer_tool.cc:1074] tablet 30620f4774b8414eb4e27bd5ecb6ecee: '7b66a8940fe54e21bec1b2db25b2fed5' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:06.352079 17303 rebalancer_tool.cc:1074] tablet cd00409d47954ee2b7555e7ff09a6e28: 'ddae2a636423489693c12b7d7d166c09' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:06.393013 17303 rebalancer_tool.cc:1084] tablet 53edbd87d0d5471ca7626ca2071f75eb: 'f77c2a969bde45b49ac1bbbe242ff171' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move ignored: Illegal state: tablet 53edbd87d0d5471ca7626ca2071f75eb: replica fc067ee690bb457e8109ad8e4eb43ab8 is already present
I20240502 01:59:06.393087 17303 rebalancer_tool.cc:756] re-synchronizing cluster state
W20240502 01:59:07.699024 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:07.699134 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server 3b8d615c71e44ff887e0f037c3efcff7 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:07.699317 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:07.699375 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0b29d58fe9c74d1cb4345bb6cfbd0535: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:07.699416 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:07.699514 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:07.699556 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:07.753489 17303 rebalancer_tool.cc:1074] tablet 8ddc239a297a46e48e069b985cb7b350: 'f77c2a969bde45b49ac1bbbe242ff171' -> '' move scheduled
I20240502 01:59:07.907333 17303 rebalancer_tool.cc:1074] tablet e2f4ca025890492880608c7975650969: '84987ad9e4a54d96af8569e4aaf0ed64' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:08.025930 17303 rebalancer_tool.cc:1084] tablet 8f2fb504a49c44edac92fd935555c278: 'ddae2a636423489693c12b7d7d166c09' -> '3b8d615c71e44ff887e0f037c3efcff7' move ignored: Illegal state: Request specified cas_config_opid_index of 6 but the committed config has opid_index of 7
I20240502 01:59:08.026021 17303 rebalancer_tool.cc:756] re-synchronizing cluster state
W20240502 01:59:08.587100 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 9736e7c5365849759564e386d208932f: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:08.587191 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 7b66a8940fe54e21bec1b2db25b2fed5
W20240502 01:59:08.588779 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 7b66a8940fe54e21bec1b2db25b2fed5
W20240502 01:59:08.588856 17303 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 9736e7c5365849759564e386d208932f: could not find any suitable replica to move from server d4e5422f66454511ae5a50ec6ab134ad to server f77c2a969bde45b49ac1bbbe242ff171
I20240502 01:59:08.860366 17303 rebalancer_tool.cc:1074] tablet 3f938c5af51a43c9a23b753c4991e41d: 'd4e5422f66454511ae5a50ec6ab134ad' -> '' move scheduled
I20240502 02:00:08.883984 17303 rebalancer_tool.cc:1084] tablet a901eb3063ea4911a822552db01d365d: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '7b66a8940fe54e21bec1b2db25b2fed5' move ignored: Timed out: BulkChangeConfig RPC to 127.7.221.66:34905 timed out after 60.000s (SENT)
I20240502 02:00:08.884083 17303 rebalancer_tool.cc:756] re-synchronizing cluster state
Illegal state: tablet server d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): unacceptable health status UNAVAILABLE
W20240502 02:00:09.198159 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:09.239456 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:09.294996 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:09.318976 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:09.345620 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:09.390115 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:00:09.390903 698 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:09.519771 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:09.542968 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:09.598934 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 94 similar messages]
W20240502 02:00:09.671454 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 103 similar messages]
W20240502 02:00:09.674477 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:09.679879 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 02:00:09.680457 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:09.686522 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:09.768640 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 02:00:09.786613 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 49 similar messages]
W20240502 02:00:09.787138 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:09.790407 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:09.795203 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:09.810379 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:09.843717 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:09.853749 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
I20240502 02:00:09.883333 698 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:09.898761 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:09.903779 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:09.950526 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:09.977388 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:09.999812 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:10.008878 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:10.062291 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:10.109777 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:10.189049 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:10.435645 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:10.465478 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:10.472268 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:00:10.503875 698 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:10.561468 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:10.613226 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:10.656351 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 02:00:10.692490 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:10.711179 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:10.736753 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:10.748358 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:10.824616 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
I20240502 02:00:10.950947 698 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:10.980962 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:10.983400 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:11.034466 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:11.086860 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:11.177702 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:11.222577 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:11.347127 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:11.400995 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:11.448766 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:11.453774 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:00:11.507577 1262 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:11.519898 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:11.529754 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:11.530220 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:11.561875 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:11.650681 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:11.829042 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:11.912382 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:11.926913 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
I20240502 02:00:11.927417 698 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:11.984004 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:12.057610 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:12.070674 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:12.091192 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:12.175693 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:12.297150 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:12.299403 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:12.309463 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:12.311093 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:12.314239 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:12.317759 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:12.341147 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:12.360843 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:12.365932 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
I20240502 02:00:12.425484 698 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:12.490893 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:12.511518 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:12.568768 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:12.600406 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:12.609284 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:12.627748 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:12.670454 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:00:12.824538 698 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:12.884490 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:12.889982 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:12.923200 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:12.995093 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:13.073287 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 02:00:13.228511 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:13.237999 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:13.254359 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:13.306941 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
I20240502 02:00:13.319555 1262 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:13.370656 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:13.409119 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:13.411175 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:13.421036 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:13.501427 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:13.523681 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:13.678298 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:13.726308 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:13.729751 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:13.798127 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:13.844579 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
I20240502 02:00:13.872475 698 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:13.888222 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:13.898844 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:13.911653 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:14.041945 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:14.243355 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:14.244683 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:14.252923 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
I20240502 02:00:14.258113 1262 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:14.321326 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:14.345126 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:14.355625 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:14.362680 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:14.512220 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:14.599047 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:14.619361 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:14.643563 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:14.683641 15398 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 02:00:14.700821 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 98 similar messages]
W20240502 02:00:14.721097 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 89 similar messages]
W20240502 02:00:14.721580 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:14.732120 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:14.785012 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 02:00:14.785759 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:14.786522 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:00:14.807546 1930 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:14.808867 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:14.815275 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:14.822193 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 49 similar messages]
W20240502 02:00:14.822753 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:14.832490 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:14.834086 14901 catalog_manager.cc:5014] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 3f938c5af51a43c9a23b753c4991e41d with cas_config_opid_index 15: ChangeConfig:ADD_PEER:NON_VOTER failed with leader 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323) due to error UNKNOWN_ERROR; will retry: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }.
Pending config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
W20240502 02:00:14.974391 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:15.025128 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:15.096477 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:15.179944 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:00:15.188635 1930 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:15.219717 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:15.300869 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:15.354108 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:15.354111 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:15.373143 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 02:00:15.386457 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:15.577929 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:15.602384 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:15.612401 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:15.641172 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:15.672418 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:15.703832 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:15.715960 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:15.741032 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
I20240502 02:00:15.743162 1930 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:15.815882 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:15.902019 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:15.945153 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:15.967041 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:16.014398 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:16.065636 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:16.129606 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:00:16.129925 1930 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:16.191594 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:16.258667 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:16.259294 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
I20240502 02:00:16.292215 2239 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:00:16.292335 2239 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:00:16.292845 2239 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:00:16.293234 15328 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:00:16.293380 15328 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:00:16.293804 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:16.294005 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:00:16.294067 15399 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:00:16.294207 2239 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:00:16.333343 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:16.438107 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:16.499361 14901 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 2 similar messages]
W20240502 02:00:16.500114 14901 catalog_manager.cc:4573] TS d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 21e2e55d1a0d4b6eb18b4ade9468607f: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:00:16.512390 2101 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:16.578671 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:16.606192 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:16.657232 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:16.705096 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:16.731382 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:16.773871 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:16.869612 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:16.881808 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:16.999878 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:17.011155 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:17.041343 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
I20240502 02:00:17.067780 2101 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:17.116667 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:17.127211 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:17.138633 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:17.218276 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:17.239311 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:17.243224 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:17.282155 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:17.290153 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:17.336629 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:17.379874 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:17.381883 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:17.488399 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:17.566511 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
I20240502 02:00:17.658164 2101 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:17.664106 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:17.690739 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:17.696936 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:17.737309 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:17.877795 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:17.918119 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:17.939190 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
I20240502 02:00:17.944576 2241 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:00:17.944696 2241 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:00:17.945190 2241 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:00:17.945504 15464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 02:00:17.945659 15464 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 02:00:17.946202 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:17.946277 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:00:17.946344 15266 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:00:17.946466 2241 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:00:18.015620 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 02:00:18.081786 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:18.089958 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:18.116510 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:18.147765 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:18.159420 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:18.204319 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:18.229539 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:18.232463 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
I20240502 02:00:18.244046 2101 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:18.326730 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:18.380967 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:18.500195 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:18.543284 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:18.655929 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:18.667917 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:18.682633 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:18.686017 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:00:18.694952 2502 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:18.723276 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:18.765923 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:18.792304 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:18.843113 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:19.088423 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:19.230827 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:19.279351 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:19.291785 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:00:19.298564 2502 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:19.395130 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:19.483906 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:19.521540 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:19.534039 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:19.579427 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:19.605600 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:19.652787 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:19.660039 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:19.668133 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:19.712832 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 82 similar messages]
W20240502 02:00:19.715787 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:19.732354 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 97 similar messages]
W20240502 02:00:19.733083 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:19.790294 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:19.790772 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 74 similar messages]
I20240502 02:00:19.854094 2502 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:19.861337 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 47 similar messages]
W20240502 02:00:19.861979 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:19.900264 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:19.901288 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:19.958206 15266 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 96 similar messages]
W20240502 02:00:19.979915 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:19.983690 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:19.990463 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:20.002216 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:20.107101 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:20.173686 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:20.188292 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:20.391866 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:20.423481 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
I20240502 02:00:20.461455 3073 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:20.479281 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:20.549094 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:20.571108 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:20.591609 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:20.624265 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 02:00:20.653260 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:20.701174 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:20.712590 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:20.756201 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:20.760754 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:20.793435 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:20.954866 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:21.039595 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:21.053483 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:21.073431 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
I20240502 02:00:21.078029 3228 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:21.095764 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:21.107360 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:21.179895 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:21.243069 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:21.268394 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:21.284632 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:21.429833 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:21.474179 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
I20240502 02:00:21.644821 3073 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:21.664983 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:00:21.700850 3032 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:00:21.700986 3032 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:00:21.701496 3032 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:00:21.701812 15328 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:00:21.701957 15328 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:00:21.702466 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:21.702685 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:00:21.702752 15396 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:00:21.702880 3032 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:00:21.732035 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:21.828784 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:21.942647 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:21.952737 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:21.967654 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:21.995085 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
I20240502 02:00:22.021097 3228 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:22.022554 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:22.056034 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:22.076766 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:22.131240 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:22.147024 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:22.164990 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:22.272014 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:22.337826 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:22.394141 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:22.399183 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:22.473709 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:22.476567 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:22.483398 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:22.507045 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:22.528384 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:22.575529 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:22.588409 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
I20240502 02:00:22.618794 3539 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:22.732337 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:22.772341 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:22.861284 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:22.885335 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:22.932889 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:23.021989 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:23.040236 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:23.084061 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:23.135126 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 02:00:23.164703 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
I20240502 02:00:23.197933 3642 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:23.221767 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:23.302735 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:23.367992 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:23.402796 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:23.442157 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:23.450362 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:23.516813 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:23.532119 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:23.592275 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:23.603904 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:23.706545 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:23.780692 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:23.787693 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:23.789304 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
I20240502 02:00:23.803414 3752 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:23.823278 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:23.846082 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:23.859798 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:24.133500 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:24.232867 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:24.322319 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
I20240502 02:00:24.372092 3752 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:24.381628 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:24.392796 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:24.465098 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:24.470041 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:24.475006 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:24.478988 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:24.625083 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:24.645798 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:24.681852 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:24.719620 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:24.742765 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:24.792467 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 101 similar messages]
W20240502 02:00:24.809509 15398 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 02:00:24.816037 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
I20240502 02:00:24.818096 3752 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:24.869668 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:24.884953 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 68 similar messages]
W20240502 02:00:24.980499 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 51 similar messages]
W20240502 02:00:24.981137 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:24.981389 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:25.025202 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:25.066766 15266 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 90 similar messages]
W20240502 02:00:25.073624 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:25.091437 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:25.092195 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:25.152107 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:25.160308 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:25.189780 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
I20240502 02:00:25.270643 3752 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:25.353832 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:25.410770 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:25.421273 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:25.581683 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:25.597280 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:25.618146 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 02:00:25.631700 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
I20240502 02:00:25.673499 3903 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:25.694682 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:25.698282 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:25.708550 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:25.766371 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:25.769906 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:25.944358 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:25.979553 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:25.981065 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:26.014106 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:26.036993 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:26.171487 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:26.174700 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:26.188548 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:26.192842 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:00:26.243624 4260 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:26.306257 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:26.462163 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:26.570545 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:26.632951 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:26.659615 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:26.727747 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:00:26.822821 4363 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:26.901290 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:26.901504 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:26.947315 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:26.969959 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:27.009783 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:27.033442 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:27.037062 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:27.106465 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:27.181241 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:27.196831 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:27.256825 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:27.270565 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:27.323136 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
I20240502 02:00:27.325718 4363 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:27.344014 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:27.357173 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:27.380115 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:27.442664 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:27.479120 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:27.480384 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:27.500839 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:27.526829 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:27.632055 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:27.642264 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:27.665943 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
I20240502 02:00:27.855255 4363 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:27.879978 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:27.898903 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:28.059370 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:28.099228 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:28.099579 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:28.132404 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:28.149147 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:28.181198 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 02:00:28.206717 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:28.212481 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:28.256351 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:28.337361 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:28.401495 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:28.403456 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
I20240502 02:00:28.478680 4726 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:28.517688 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:28.531343 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:28.573295 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:28.621724 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:28.659832 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:28.716148 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:28.718704 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:28.768343 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:28.877735 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
I20240502 02:00:29.016659 4726 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:29.115284 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:29.115770 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:29.137475 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:29.152668 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:29.239249 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:29.260414 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:29.360670 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:29.409854 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:29.572819 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:29.582692 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:29.586515 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:29.595901 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
I20240502 02:00:29.626055 4726 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:29.660246 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:29.688760 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:29.756528 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
I20240502 02:00:29.814379 4669 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:00:29.814510 4669 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:00:29.815110 4669 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:00:29.815393 15464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 02:00:29.815557 15464 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 02:00:29.816161 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:29.816354 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 02:00:29.816524 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:00:29.816594 15266 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:00:29.816733 4669 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:00:29.844045 14992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 96 similar messages]
W20240502 02:00:29.844695 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:29.862052 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:29.887923 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 70 similar messages]
W20240502 02:00:29.912091 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:29.926537 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:29.934343 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:29.969190 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:30.008562 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 02:00:30.034368 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:30.076764 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:30.092226 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:30.128216 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 91 similar messages]
I20240502 02:00:30.135243 5053 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:30.194808 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:30.217752 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:30.228178 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:30.360462 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:30.363549 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:30.364125 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:30.395205 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 02:00:30.515615 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:00:30.577593 4726 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:30.578930 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:30.633163 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:30.682638 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:30.695847 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:30.787479 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:30.798866 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:30.841634 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:30.922825 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:30.953090 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:31.014461 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:31.030560 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:31.068928 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:31.099505 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:31.112874 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:31.169119 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:31.171676 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:31.172055 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
I20240502 02:00:31.190254 5291 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:31.217475 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:31.431324 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:31.502566 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:31.617959 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
I20240502 02:00:31.716280 5394 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:31.744835 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:31.758340 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:31.775835 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:31.802317 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:31.842258 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:31.883713 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:31.921880 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:31.930691 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:32.009474 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:32.029734 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:32.137719 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:32.159448 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
I20240502 02:00:32.204192 5291 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:32.239814 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:32.272262 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:32.282858 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:32.339924 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:32.368680 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:32.392088 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:32.420159 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:32.479575 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:32.532804 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:32.539613 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:32.686177 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:32.776810 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:32.785233 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:32.786144 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
I20240502 02:00:32.823988 5291 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:32.828939 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:32.875038 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:32.875082 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:32.930959 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 02:00:32.992193 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:33.036742 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:33.061386 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:33.064944 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:33.145366 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:33.159914 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:33.224762 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:33.269817 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:33.389189 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
I20240502 02:00:33.389757 5704 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:33.393266 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:33.409850 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:33.422396 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:33.604249 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:33.619637 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:33.668370 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:33.769192 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:33.774585 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:33.791519 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:33.801363 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:33.959839 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
I20240502 02:00:33.971500 5851 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:34.060878 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:34.097811 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:34.146975 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:34.166827 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:34.184156 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:34.191216 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:34.220997 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:34.318909 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:34.373103 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:34.420115 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
I20240502 02:00:34.421845 5851 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:34.479099 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:34.511466 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:34.546628 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:34.680867 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:34.681351 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:34.708113 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:34.715735 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:34.750325 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:34.848630 14994 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 100 similar messages]
W20240502 02:00:34.930202 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 69 similar messages]
I20240502 02:00:34.941159 5851 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:34.972999 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:34.987406 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 83 similar messages]
W20240502 02:00:35.056749 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:35.067617 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 02:00:35.076392 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:35.129287 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 93 similar messages]
W20240502 02:00:35.129868 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:35.132550 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 51 similar messages]
W20240502 02:00:35.185621 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:35.202049 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:35.217446 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:35.257306 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:35.272634 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:35.301656 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:35.368232 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:35.404600 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:35.410463 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
I20240502 02:00:35.425750 6015 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:35.558097 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:35.576954 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:35.627283 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:35.665031 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:35.679872 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:35.734097 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:35.760819 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:35.793349 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:35.811679 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:35.835418 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:35.865192 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
I20240502 02:00:36.040685 6015 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:36.122231 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:36.124284 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:36.248932 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:36.253597 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:36.273192 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:36.273591 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:36.470129 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:36.481096 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:36.492887 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:36.525341 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:36.558566 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:36.593695 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:36.637843 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:36.638161 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
I20240502 02:00:36.663151 6374 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:36.769454 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:36.791206 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:36.848147 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:36.897601 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:36.980706 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:37.080226 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:37.105360 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:37.144922 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:37.170364 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:00:37.227532 6015 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:37.239022 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:37.248751 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:37.340143 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:37.458149 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:37.518913 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:37.533354 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:37.620910 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:37.625192 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:37.662456 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:37.710139 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:37.738310 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:37.767931 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:37.803936 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
I20240502 02:00:37.825439 6374 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:37.860473 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:37.909032 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:37.990191 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:38.036103 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:38.048995 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:38.058475 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:38.109848 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:38.216872 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:38.248571 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:38.249147 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 02:00:38.275095 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
I20240502 02:00:38.281847 6374 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:38.309355 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:38.392585 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:38.439711 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:38.527737 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:38.563539 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:38.582785 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:38.612190 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:38.673063 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:00:38.705364 6374 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:38.715231 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:38.859292 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:38.928133 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:38.966560 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:38.996706 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:39.037467 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:39.068781 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:39.109186 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:39.162108 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:39.203157 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
I20240502 02:00:39.260838 6374 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:39.261761 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:39.294970 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:39.357277 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:39.396148 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:39.449299 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:39.455423 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:39.475170 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:39.614084 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:39.701762 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:39.716604 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
I20240502 02:00:39.735328 6374 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:39.782681 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:39.826643 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:39.857407 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 02:00:39.904489 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 101 similar messages]
W20240502 02:00:39.905162 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:39.941614 15127 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 71 similar messages]
W20240502 02:00:39.953446 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:39.999663 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:40.014026 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 02:00:40.014690 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:40.089826 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:40.174414 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 49 similar messages]
W20240502 02:00:40.230525 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 88 similar messages]
W20240502 02:00:40.233024 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
I20240502 02:00:40.270902 6374 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:40.334652 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:40.381145 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:40.392874 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:40.427897 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:40.475718 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:40.511875 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:40.521288 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:40.537590 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:40.660878 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:40.718627 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:40.721315 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:40.746269 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:40.760030 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:40.783335 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
I20240502 02:00:40.790179 6374 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:40.876698 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:40.903829 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:40.947924 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:40.970189 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:41.031708 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:41.042435 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:41.127200 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:41.273566 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
I20240502 02:00:41.275174 6374 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:41.275544 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:41.383749 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:41.417285 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:41.438489 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:41.461897 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:41.508370 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:41.547032 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:41.575523 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
I20240502 02:00:41.667511 5912 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:00:41.667649 5912 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:00:41.668135 7219 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 02:00:41.668277 5912 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:00:41.669000 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:00:41.669140 15329 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:00:41.669507 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:41.669600 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:00:41.669674 15399 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:00:41.669827 5912 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:00:41.686231 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:41.732224 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:41.787029 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:41.815686 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:41.937816 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:41.954635 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:42.063560 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:42.066509 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
I20240502 02:00:42.075707 7219 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:42.140275 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:42.159322 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:42.160125 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:42.228070 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:42.288513 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:42.399552 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:42.420281 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
I20240502 02:00:42.488361 7219 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:42.499861 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:42.518718 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:42.556742 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:42.560530 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:42.570092 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:42.728195 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:42.791023 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:42.803208 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
I20240502 02:00:42.899879 6374 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:42.914647 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:42.961721 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:43.028573 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:43.047433 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:43.047873 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:43.048588 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:43.074298 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:43.083878 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:43.200464 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:43.221693 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:43.339471 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
I20240502 02:00:43.351625 6374 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:43.406827 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:43.492825 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:43.508237 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:43.521863 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:43.569591 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 02:00:43.590416 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:43.632928 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:43.696120 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:43.767175 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:43.823405 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:43.832207 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:43.889659 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:43.907253 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:43.913482 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:00:43.953826 7858 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:43.990795 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:44.041822 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:44.063419 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:44.192844 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:44.319409 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:44.404069 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:44.438920 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:44.441578 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:44.446252 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:44.484947 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
I20240502 02:00:44.558585 7858 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:44.628113 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:44.641644 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:44.669821 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:44.673578 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:44.710575 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:44.726913 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:44.850705 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:44.853914 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:44.859377 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:44.887897 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:44.905891 14992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 97 similar messages]
W20240502 02:00:44.914721 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:44.919884 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:45.013223 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 71 similar messages]
I20240502 02:00:45.048096 7858 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:45.099117 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 02:00:45.228354 16773 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 02:00:45.248986 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 88 similar messages]
W20240502 02:00:45.407763 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:45.409544 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:45.423890 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:45.434979 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:45.458009 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:45.459535 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:45.515309 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:45.517985 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:45.529495 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
I20240502 02:00:45.613350 8219 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:45.635591 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:45.681433 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:45.711198 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:45.775732 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:45.793483 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:45.843932 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:45.845934 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:45.873721 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:45.879667 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:45.924585 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:45.939380 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:45.988112 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 02:00:46.139024 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
I20240502 02:00:46.191521 8219 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:46.199780 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:46.211405 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:46.214644 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:46.303138 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:46.317595 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:46.407176 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:46.428607 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:46.463331 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:46.547348 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:46.612237 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
I20240502 02:00:46.621785 8327 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:46.675458 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:46.791328 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:46.915524 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:46.945148 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:46.948458 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:46.971830 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:46.992391 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:46.997107 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
I20240502 02:00:47.084297 8327 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:47.120203 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:47.203626 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:47.210065 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:47.237571 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:47.241398 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:47.330809 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:47.350239 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:47.352574 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:47.392223 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:47.402333 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:00:47.497959 8327 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:47.534267 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:47.788553 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:47.793798 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:47.796494 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:47.831326 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:47.846127 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:47.890520 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:47.892704 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:47.930467 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:47.935379 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
I20240502 02:00:48.078907 8691 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 02:00:48.120184 8275 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:00:48.120292 8275 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:00:48.120859 8275 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:00:48.121444 15464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 02:00:48.121603 15464 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 02:00:48.121948 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:48.122382 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:00:48.122452 15266 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:00:48.122679 8275 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:00:48.178261 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:48.239787 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:48.260903 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:48.267596 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:48.278131 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:48.294726 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:48.315765 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 02:00:48.332242 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:48.458207 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:48.461143 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:48.519465 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:48.620569 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
I20240502 02:00:48.651901 8691 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:48.701038 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:48.704754 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:48.784749 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:48.793803 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:48.833717 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:48.841689 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:48.900063 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:48.917542 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:48.935443 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:48.957754 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:49.105826 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
I20240502 02:00:49.122623 8691 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:49.140969 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:49.249622 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:49.418730 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:49.450423 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:49.488462 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:49.536404 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:49.556260 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:49.563802 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:49.703879 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:49.724767 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
I20240502 02:00:49.726555 8691 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:49.752614 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:49.777909 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:49.829594 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:49.850250 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:49.909736 14992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 98 similar messages]
W20240502 02:00:49.910426 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:49.960563 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:49.962848 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:50.025246 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:50.119194 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 68 similar messages]
W20240502 02:00:50.128093 15398 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 78 similar messages]
W20240502 02:00:50.194764 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:50.243232 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 49 similar messages]
W20240502 02:00:50.243800 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:50.255650 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 91 similar messages]
I20240502 02:00:50.265084 9058 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:50.280473 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:50.316177 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:50.362733 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:50.427094 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:50.463479 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:50.523643 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:50.564319 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:50.582767 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:50.603132 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:50.654314 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:50.660805 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20240502 02:00:50.741325 9058 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:50.748574 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:50.765195 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 02:00:50.796708 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:50.813511 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:51.076804 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:51.109189 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:51.117409 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:51.122330 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:51.124743 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20240502 02:00:51.146512 8691 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:51.236640 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:51.269757 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:51.298347 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:51.319826 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:51.320549 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:51.387890 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:51.430732 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:51.549070 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:51.556567 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:51.601087 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:51.723157 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:51.748957 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
I20240502 02:00:51.762291 8691 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:51.831817 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:00:51.976392 14904 maintenance_manager.cc:419] P 0d981b1e50234e83abbb17940d1327fd: Scheduling FlushMRSOp(00000000000000000000000000000000): perf score=0.033369
W20240502 02:00:51.991463 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20240502 02:00:51.995442 14903 maintenance_manager.cc:643] P 0d981b1e50234e83abbb17940d1327fd: FlushMRSOp(00000000000000000000000000000000) complete. Timing: real 0.019s user 0.017s sys 0.001s Metrics: {"bytes_written":21305,"cfile_init":1,"dirs.queue_time_us":162,"dirs.run_cpu_time_us":176,"dirs.run_wall_time_us":1097,"drs_written":1,"lbm_read_time_us":72,"lbm_reads_lt_1ms":4,"lbm_write_time_us":581,"lbm_writes_lt_1ms":29,"peak_mem_usage":0,"rows_written":43,"thread_start_us":122,"threads_started":2,"wal-append.queue_time_us":114}
I20240502 02:00:51.996228 14904 maintenance_manager.cc:419] P 0d981b1e50234e83abbb17940d1327fd: Scheduling UndoDeltaBlockGCOp(00000000000000000000000000000000): 11903 bytes on disk
I20240502 02:00:51.996686 14903 maintenance_manager.cc:643] P 0d981b1e50234e83abbb17940d1327fd: UndoDeltaBlockGCOp(00000000000000000000000000000000) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":46,"lbm_reads_lt_1ms":4}
W20240502 02:00:52.007851 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:52.035642 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:52.103765 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:52.132453 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:52.149416 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:52.158241 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:52.252548 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:52.283293 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:52.335837 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:52.350720 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20240502 02:00:52.370565 9575 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:52.376646 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:52.512583 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:52.540642 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:52.558243 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:52.604516 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:52.614724 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:52.706425 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:52.805050 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:52.806150 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:52.822394 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:52.861732 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:52.933681 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:52.943326 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
I20240502 02:00:52.964455 9727 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:53.093078 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:53.172497 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:53.173516 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 02:00:53.256057 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:53.286371 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:53.341806 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:53.365625 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:53.381326 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:53.387374 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20240502 02:00:53.481600 9834 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:53.523888 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:53.539430 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:53.668813 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:53.710824 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:53.796820 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:53.804492 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:53.859606 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:53.952535 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:53.954908 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
I20240502 02:00:53.968401 9727 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:53.983650 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:54.003374 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:54.042690 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:54.062361 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:54.074232 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:54.166251 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:54.199548 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:54.201967 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:54.360745 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:54.440346 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:54.463832 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:54.482813 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:54.526132 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
I20240502 02:00:54.553916 10041 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:54.572031 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:54.642051 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:54.671299 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:54.809644 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:54.830045 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:54.836845 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:54.868765 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:54.929431 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:54.949975 14994 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 97 similar messages]
W20240502 02:00:54.950170 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 02:00:54.969235 10042 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:54.979748 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:55.047312 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:55.136900 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 02:00:55.137545 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:55.144209 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:55.196470 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:55.230609 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:55.235590 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 71 similar messages]
W20240502 02:00:55.323400 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 02:00:55.402807 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 51 similar messages]
W20240502 02:00:55.407361 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:55.420464 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
I20240502 02:00:55.443903 9727 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:55.448096 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:55.517747 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:55.535272 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:55.603191 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 02:00:55.610129 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:55.618667 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:55.663527 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:55.681421 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:55.740756 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:55.755724 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:55.857650 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:55.875327 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
I20240502 02:00:55.888118 10042 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:55.934250 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:56.159716 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:56.163515 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:56.248572 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:56.292102 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:56.321520 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:56.349397 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:56.355258 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:56.374037 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 02:00:56.411859 10402 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:56.469157 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:56.495786 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:56.506745 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:56.575296 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:56.611029 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:00:56.747856 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:00:56.840348 10402 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:56.843400 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:56.960261 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:56.981356 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:57.011406 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:57.030548 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:57.048460 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:57.142499 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 02:00:57.150803 9059 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:00:57.150976 9059 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:00:57.151459 9059 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:00:57.151798 15464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 02:00:57.151981 15464 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 02:00:57.152889 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:57.153029 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:00:57.153087 15266 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:00:57.153225 9059 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
I20240502 02:00:57.158381 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling FlushMRSOp(21e2e55d1a0d4b6eb18b4ade9468607f): perf score=0.033342
I20240502 02:00:57.165696 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling FlushMRSOp(e2f4ca025890492880608c7975650969): perf score=0.033345
W20240502 02:00:57.172315 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 02:00:57.191895 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: FlushMRSOp(21e2e55d1a0d4b6eb18b4ade9468607f) complete. Timing: real 0.033s user 0.031s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"compiler_manager_pool.queue_time_us":170,"drs_written":1,"lbm_read_time_us":69,"lbm_reads_lt_1ms":4,"lbm_write_time_us":527,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":240,"threads_started":3,"wal-append.queue_time_us":173}
I20240502 02:00:57.193363 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(21e2e55d1a0d4b6eb18b4ade9468607f): 4921 bytes on disk
I20240502 02:00:57.193821 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(21e2e55d1a0d4b6eb18b4ade9468607f) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":44,"lbm_reads_lt_1ms":4}
I20240502 02:00:57.199820 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: FlushMRSOp(e2f4ca025890492880608c7975650969) complete. Timing: real 0.034s user 0.030s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"compiler_manager_pool.queue_time_us":169,"dirs.queue_time_us":2037,"dirs.run_cpu_time_us":161,"dirs.run_wall_time_us":1982,"drs_written":1,"lbm_read_time_us":67,"lbm_reads_lt_1ms":4,"lbm_write_time_us":572,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":261,"threads_started":3,"wal-append.queue_time_us":160}
I20240502 02:00:57.200659 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(fcfc8dc4b90a4a6aa7e4035bbfdb4532): perf score=0.033357
I20240502 02:00:57.200912 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling UndoDeltaBlockGCOp(e2f4ca025890492880608c7975650969): 4581 bytes on disk
I20240502 02:00:57.201485 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: UndoDeltaBlockGCOp(e2f4ca025890492880608c7975650969) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":63,"lbm_reads_lt_1ms":4}
I20240502 02:00:57.230791 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling FlushMRSOp(e2f4ca025890492880608c7975650969): perf score=0.033364
I20240502 02:00:57.264792 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(fcfc8dc4b90a4a6aa7e4035bbfdb4532) complete. Timing: real 0.064s user 0.031s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"compiler_manager_pool.queue_time_us":519,"drs_written":1,"lbm_read_time_us":64,"lbm_reads_lt_1ms":4,"lbm_write_time_us":529,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":271,"threads_started":3,"wal-append.queue_time_us":6553}
I20240502 02:00:57.268523 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(fcfc8dc4b90a4a6aa7e4035bbfdb4532): 4834 bytes on disk
I20240502 02:00:57.270905 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(fcfc8dc4b90a4a6aa7e4035bbfdb4532) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":94,"lbm_reads_lt_1ms":4}
I20240502 02:00:57.276695 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(21e2e55d1a0d4b6eb18b4ade9468607f): perf score=0.033377
W20240502 02:00:57.287955 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:57.297194 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 02:00:57.335724 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: FlushMRSOp(e2f4ca025890492880608c7975650969) complete. Timing: real 0.105s user 0.031s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"compiler_manager_pool.queue_time_us":19495,"dirs.queue_time_us":5325,"dirs.run_cpu_time_us":163,"dirs.run_wall_time_us":2742,"drs_written":1,"lbm_read_time_us":96,"lbm_reads_lt_1ms":4,"lbm_write_time_us":678,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":262,"threads_started":3,"wal-append.queue_time_us":5868}
I20240502 02:00:57.336882 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling UndoDeltaBlockGCOp(e2f4ca025890492880608c7975650969): 4581 bytes on disk
I20240502 02:00:57.340761 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: UndoDeltaBlockGCOp(e2f4ca025890492880608c7975650969) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":53,"lbm_reads_lt_1ms":4}
I20240502 02:00:57.354036 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(21e2e55d1a0d4b6eb18b4ade9468607f) complete. Timing: real 0.077s user 0.019s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":42,"dirs.run_cpu_time_us":172,"dirs.run_wall_time_us":1148,"drs_written":1,"lbm_read_time_us":52,"lbm_reads_lt_1ms":4,"lbm_write_time_us":524,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":79,"threads_started":1,"wal-append.queue_time_us":6249}
I20240502 02:00:57.355259 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(21e2e55d1a0d4b6eb18b4ade9468607f): 4921 bytes on disk
I20240502 02:00:57.355744 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(21e2e55d1a0d4b6eb18b4ade9468607f) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":68,"lbm_reads_lt_1ms":4}
W20240502 02:00:57.389803 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:57.425191 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:57.456290 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:00:57.458321 10402 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:57.464962 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:57.507556 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:57.515437 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:00:57.568367 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:00:57.575881 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:57.626274 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:00:57.696312 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling FlushMRSOp(8ddc239a297a46e48e069b985cb7b350): perf score=0.033336
I20240502 02:00:57.703716 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling FlushMRSOp(53edbd87d0d5471ca7626ca2071f75eb): perf score=0.033338
I20240502 02:00:57.712972 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: FlushMRSOp(8ddc239a297a46e48e069b985cb7b350) complete. Timing: real 0.016s user 0.014s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":4102,"dirs.run_cpu_time_us":171,"dirs.run_wall_time_us":1623,"drs_written":1,"lbm_read_time_us":55,"lbm_reads_lt_1ms":4,"lbm_write_time_us":535,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":156,"threads_started":2,"wal-append.queue_time_us":123}
I20240502 02:00:57.713856 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(8ddc239a297a46e48e069b985cb7b350): 4834 bytes on disk
I20240502 02:00:57.714282 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(8ddc239a297a46e48e069b985cb7b350) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4}
I20240502 02:00:57.718852 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: FlushMRSOp(53edbd87d0d5471ca7626ca2071f75eb) complete. Timing: real 0.015s user 0.014s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":130,"dirs.run_cpu_time_us":171,"dirs.run_wall_time_us":1263,"drs_written":1,"lbm_read_time_us":55,"lbm_reads_lt_1ms":4,"lbm_write_time_us":572,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"spinlock_wait_cycles":3456,"thread_start_us":110,"threads_started":2,"wal-append.queue_time_us":117}
I20240502 02:00:57.719579 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling UndoDeltaBlockGCOp(53edbd87d0d5471ca7626ca2071f75eb): 4581 bytes on disk
I20240502 02:00:57.719884 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: UndoDeltaBlockGCOp(53edbd87d0d5471ca7626ca2071f75eb) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":45,"lbm_reads_lt_1ms":4}
I20240502 02:00:57.720435 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling FlushMRSOp(8ddc239a297a46e48e069b985cb7b350): perf score=0.033342
W20240502 02:00:57.723845 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 02:00:57.732264 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: FlushMRSOp(8ddc239a297a46e48e069b985cb7b350) complete. Timing: real 0.012s user 0.010s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":24,"dirs.run_cpu_time_us":134,"dirs.run_wall_time_us":1145,"drs_written":1,"lbm_read_time_us":50,"lbm_reads_lt_1ms":4,"lbm_write_time_us":474,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":65,"threads_started":1,"wal-append.queue_time_us":129}
I20240502 02:00:57.732879 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling UndoDeltaBlockGCOp(8ddc239a297a46e48e069b985cb7b350): 4834 bytes on disk
I20240502 02:00:57.733167 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: UndoDeltaBlockGCOp(8ddc239a297a46e48e069b985cb7b350) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":42,"lbm_reads_lt_1ms":4}
W20240502 02:00:57.831214 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20240502 02:00:57.848259 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling FlushMRSOp(fc07e430e3e74e3885f263881bd04976): perf score=0.033377
I20240502 02:00:57.858287 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(fc07e430e3e74e3885f263881bd04976): perf score=0.033383
I20240502 02:00:57.860060 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: FlushMRSOp(fc07e430e3e74e3885f263881bd04976) complete. Timing: real 0.012s user 0.010s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":163,"dirs.run_cpu_time_us":188,"dirs.run_wall_time_us":1362,"drs_written":1,"lbm_read_time_us":53,"lbm_reads_lt_1ms":4,"lbm_write_time_us":405,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":133,"threads_started":2,"wal-append.queue_time_us":137}
I20240502 02:00:57.860755 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling UndoDeltaBlockGCOp(fc07e430e3e74e3885f263881bd04976): 4921 bytes on disk
I20240502 02:00:57.861083 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: UndoDeltaBlockGCOp(fc07e430e3e74e3885f263881bd04976) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":53,"lbm_reads_lt_1ms":4}
W20240502 02:00:57.866231 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:00:57.870075 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(fc07e430e3e74e3885f263881bd04976) complete. Timing: real 0.012s user 0.008s sys 0.002s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":192,"dirs.run_cpu_time_us":209,"dirs.run_wall_time_us":1222,"drs_written":1,"lbm_read_time_us":37,"lbm_reads_lt_1ms":4,"lbm_write_time_us":471,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":169,"threads_started":2,"wal-append.queue_time_us":145}
I20240502 02:00:57.870882 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(53edbd87d0d5471ca7626ca2071f75eb): perf score=0.033385
W20240502 02:00:57.873019 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:00:57.880964 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(53edbd87d0d5471ca7626ca2071f75eb) complete. Timing: real 0.010s user 0.008s sys 0.001s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":117,"dirs.run_cpu_time_us":127,"dirs.run_wall_time_us":1098,"drs_written":1,"lbm_read_time_us":32,"lbm_reads_lt_1ms":4,"lbm_write_time_us":386,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":100,"threads_started":1,"wal-append.queue_time_us":167}
I20240502 02:00:57.881754 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(fc07e430e3e74e3885f263881bd04976): 4921 bytes on disk
I20240502 02:00:57.882067 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(fc07e430e3e74e3885f263881bd04976) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":50,"lbm_reads_lt_1ms":4}
I20240502 02:00:57.882705 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(53edbd87d0d5471ca7626ca2071f75eb): 4581 bytes on disk
I20240502 02:00:57.883009 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(53edbd87d0d5471ca7626ca2071f75eb) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":42,"lbm_reads_lt_1ms":4}
W20240502 02:00:57.963095 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 02:00:57.997653 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:58.004427 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:00:58.039188 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 02:00:58.064483 10733 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:58.079615 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:58.149643 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:58.152823 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:58.239197 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:58.263691 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:58.312391 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:58.339075 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:58.346715 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:58.395527 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 02:00:58.467268 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling FlushMRSOp(bf73e677bcd049869cadf9f8c9cd3434): perf score=0.033352
I20240502 02:00:58.482309 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: FlushMRSOp(bf73e677bcd049869cadf9f8c9cd3434) complete. Timing: real 0.015s user 0.012s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":148,"dirs.run_cpu_time_us":180,"dirs.run_wall_time_us":1376,"drs_written":1,"lbm_read_time_us":35,"lbm_reads_lt_1ms":4,"lbm_write_time_us":421,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":147,"threads_started":2,"wal-append.queue_time_us":156}
I20240502 02:00:58.483240 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(bf73e677bcd049869cadf9f8c9cd3434): 4921 bytes on disk
I20240502 02:00:58.483626 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(bf73e677bcd049869cadf9f8c9cd3434) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":48,"lbm_reads_lt_1ms":4}
I20240502 02:00:58.485693 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling FlushMRSOp(bf73e677bcd049869cadf9f8c9cd3434): perf score=0.033355
I20240502 02:00:58.495901 10733 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 02:00:58.496865 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: FlushMRSOp(bf73e677bcd049869cadf9f8c9cd3434) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":147,"dirs.run_cpu_time_us":145,"dirs.run_wall_time_us":1277,"drs_written":1,"lbm_read_time_us":37,"lbm_reads_lt_1ms":4,"lbm_write_time_us":406,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":133,"threads_started":2,"wal-append.queue_time_us":132}
I20240502 02:00:58.497619 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling UndoDeltaBlockGCOp(bf73e677bcd049869cadf9f8c9cd3434): 4921 bytes on disk
I20240502 02:00:58.497992 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: UndoDeltaBlockGCOp(bf73e677bcd049869cadf9f8c9cd3434) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":44,"lbm_reads_lt_1ms":4}
I20240502 02:00:58.613303 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling FlushMRSOp(986e7c97e0734e958f364dc5d1167846): perf score=0.033391
W20240502 02:00:58.623656 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20240502 02:00:58.627758 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: FlushMRSOp(986e7c97e0734e958f364dc5d1167846) complete. Timing: real 0.014s user 0.013s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":170,"dirs.run_cpu_time_us":205,"dirs.run_wall_time_us":1745,"drs_written":1,"lbm_read_time_us":49,"lbm_reads_lt_1ms":4,"lbm_write_time_us":482,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":165,"threads_started":2,"wal-append.queue_time_us":142}
I20240502 02:00:58.628480 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling UndoDeltaBlockGCOp(986e7c97e0734e958f364dc5d1167846): 4579 bytes on disk
I20240502 02:00:58.628824 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: UndoDeltaBlockGCOp(986e7c97e0734e958f364dc5d1167846) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":52,"lbm_reads_lt_1ms":4}
I20240502 02:00:58.635605 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(bf73e677bcd049869cadf9f8c9cd3434): perf score=0.033398
I20240502 02:00:58.646711 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(bf73e677bcd049869cadf9f8c9cd3434) complete. Timing: real 0.011s user 0.011s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":175,"dirs.run_cpu_time_us":116,"dirs.run_wall_time_us":1062,"drs_written":1,"lbm_read_time_us":37,"lbm_reads_lt_1ms":4,"lbm_write_time_us":362,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":153,"threads_started":2,"wal-append.queue_time_us":130}
I20240502 02:00:58.647527 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(986e7c97e0734e958f364dc5d1167846): perf score=0.033401
W20240502 02:00:58.656719 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:00:58.658298 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(986e7c97e0734e958f364dc5d1167846) complete. Timing: real 0.011s user 0.009s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":39,"dirs.run_cpu_time_us":156,"dirs.run_wall_time_us":1043,"drs_written":1,"lbm_read_time_us":38,"lbm_reads_lt_1ms":4,"lbm_write_time_us":389,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":77,"threads_started":1,"wal-append.queue_time_us":152}
I20240502 02:00:58.659145 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(bf73e677bcd049869cadf9f8c9cd3434): 4921 bytes on disk
I20240502 02:00:58.659463 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(bf73e677bcd049869cadf9f8c9cd3434) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":44,"lbm_reads_lt_1ms":4}
I20240502 02:00:58.660152 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(3f938c5af51a43c9a23b753c4991e41d): perf score=0.033403
I20240502 02:00:58.670291 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(3f938c5af51a43c9a23b753c4991e41d) complete. Timing: real 0.010s user 0.009s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":46,"dirs.run_cpu_time_us":106,"dirs.run_wall_time_us":1057,"drs_written":1,"lbm_read_time_us":33,"lbm_reads_lt_1ms":4,"lbm_write_time_us":412,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":99,"threads_started":1,"wal-append.queue_time_us":169}
I20240502 02:00:58.671090 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(3f938c5af51a43c9a23b753c4991e41d): 4834 bytes on disk
I20240502 02:00:58.671389 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(3f938c5af51a43c9a23b753c4991e41d) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":45,"lbm_reads_lt_1ms":4}
I20240502 02:00:58.672039 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(986e7c97e0734e958f364dc5d1167846): 4579 bytes on disk
I20240502 02:00:58.672322 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(986e7c97e0734e958f364dc5d1167846) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":42,"lbm_reads_lt_1ms":4}
W20240502 02:00:58.680779 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:58.689065 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:00:58.747895 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:58.757398 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:58.861233 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:00:58.861604 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:00:58.979451 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:00:59.073787 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:00:59.100076 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20240502 02:00:59.114571 10946 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 02:00:59.130739 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling FlushMRSOp(6db29f2516e1486782be0312c9bcac8b): perf score=0.033368
I20240502 02:00:59.143500 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: FlushMRSOp(6db29f2516e1486782be0312c9bcac8b) complete. Timing: real 0.012s user 0.012s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":182,"dirs.run_cpu_time_us":171,"dirs.run_wall_time_us":1419,"drs_written":1,"lbm_read_time_us":35,"lbm_reads_lt_1ms":4,"lbm_write_time_us":436,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":165,"threads_started":2,"wal-append.queue_time_us":129}
I20240502 02:00:59.144392 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling UndoDeltaBlockGCOp(6db29f2516e1486782be0312c9bcac8b): 4921 bytes on disk
I20240502 02:00:59.144721 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: UndoDeltaBlockGCOp(6db29f2516e1486782be0312c9bcac8b) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":46,"lbm_reads_lt_1ms":4}
I20240502 02:00:59.145367 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling FlushMRSOp(091d27d51acc4db0a1c7d20deeff5c10): perf score=0.033371
I20240502 02:00:59.157070 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: FlushMRSOp(091d27d51acc4db0a1c7d20deeff5c10) complete. Timing: real 0.012s user 0.010s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":43,"dirs.run_cpu_time_us":156,"dirs.run_wall_time_us":1213,"drs_written":1,"lbm_read_time_us":32,"lbm_reads_lt_1ms":4,"lbm_write_time_us":380,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":75,"threads_started":1,"wal-append.queue_time_us":145}
I20240502 02:00:59.157828 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling UndoDeltaBlockGCOp(091d27d51acc4db0a1c7d20deeff5c10): 4581 bytes on disk
I20240502 02:00:59.158210 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: UndoDeltaBlockGCOp(091d27d51acc4db0a1c7d20deeff5c10) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":48,"lbm_reads_lt_1ms":4}
W20240502 02:00:59.162897 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 02:00:59.174470 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(5804597cc507483dba622e6ec44e5ead): perf score=0.033380
I20240502 02:00:59.186708 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(5804597cc507483dba622e6ec44e5ead) complete. Timing: real 0.012s user 0.012s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":173,"dirs.run_cpu_time_us":221,"dirs.run_wall_time_us":1279,"drs_written":1,"lbm_read_time_us":74,"lbm_reads_lt_1ms":4,"lbm_write_time_us":440,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":166,"threads_started":2,"wal-append.queue_time_us":187}
I20240502 02:00:59.187530 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(5804597cc507483dba622e6ec44e5ead): 4834 bytes on disk
I20240502 02:00:59.187857 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(5804597cc507483dba622e6ec44e5ead) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4}
W20240502 02:00:59.209571 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:59.226483 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:00:59.231772 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20240502 02:00:59.236585 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling FlushMRSOp(6db29f2516e1486782be0312c9bcac8b): perf score=0.033399
I20240502 02:00:59.251020 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling FlushMRSOp(091d27d51acc4db0a1c7d20deeff5c10): perf score=0.033402
I20240502 02:00:59.253239 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: FlushMRSOp(6db29f2516e1486782be0312c9bcac8b) complete. Timing: real 0.016s user 0.015s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":152,"dirs.run_cpu_time_us":198,"dirs.run_wall_time_us":1383,"drs_written":1,"lbm_read_time_us":40,"lbm_reads_lt_1ms":4,"lbm_write_time_us":666,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":153,"threads_started":2,"wal-append.queue_time_us":179}
I20240502 02:00:59.253995 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(6db29f2516e1486782be0312c9bcac8b): 4921 bytes on disk
I20240502 02:00:59.254309 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(6db29f2516e1486782be0312c9bcac8b) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":45,"lbm_reads_lt_1ms":4}
I20240502 02:00:59.255165 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling FlushMRSOp(5804597cc507483dba622e6ec44e5ead): perf score=0.033403
W20240502 02:00:59.261636 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20240502 02:00:59.263403 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: FlushMRSOp(091d27d51acc4db0a1c7d20deeff5c10) complete. Timing: real 0.012s user 0.007s sys 0.004s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":148,"dirs.run_cpu_time_us":148,"dirs.run_wall_time_us":1163,"drs_written":1,"lbm_read_time_us":33,"lbm_reads_lt_1ms":4,"lbm_write_time_us":446,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":141,"threads_started":2,"wal-append.queue_time_us":140}
I20240502 02:00:59.264463 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling UndoDeltaBlockGCOp(091d27d51acc4db0a1c7d20deeff5c10): 4581 bytes on disk
I20240502 02:00:59.264813 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: UndoDeltaBlockGCOp(091d27d51acc4db0a1c7d20deeff5c10) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":48,"lbm_reads_lt_1ms":4}
I20240502 02:00:59.267798 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: FlushMRSOp(5804597cc507483dba622e6ec44e5ead) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":49,"dirs.run_cpu_time_us":169,"dirs.run_wall_time_us":1082,"drs_written":1,"lbm_read_time_us":55,"lbm_reads_lt_1ms":4,"lbm_write_time_us":428,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":95,"threads_started":1,"wal-append.queue_time_us":198}
I20240502 02:00:59.268522 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(5804597cc507483dba622e6ec44e5ead): 4834 bytes on disk
I20240502 02:00:59.268831 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(5804597cc507483dba622e6ec44e5ead) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":44,"lbm_reads_lt_1ms":4}
W20240502 02:00:59.344605 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:59.455122 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:00:59.516237 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling FlushMRSOp(1eb15147ff6c4d0b83b12dec736fc576): perf score=0.033350
I20240502 02:00:59.520222 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling FlushMRSOp(f97c5c9bb4c84957857efc76d1c7717f): perf score=0.033349
I20240502 02:00:59.528589 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: FlushMRSOp(1eb15147ff6c4d0b83b12dec736fc576) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":51,"dirs.run_cpu_time_us":158,"dirs.run_wall_time_us":1618,"drs_written":1,"lbm_read_time_us":57,"lbm_reads_lt_1ms":4,"lbm_write_time_us":413,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":64,"threads_started":1,"wal-append.queue_time_us":157}
I20240502 02:00:59.529417 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling FlushMRSOp(f97c5c9bb4c84957857efc76d1c7717f): perf score=0.033353
I20240502 02:00:59.532526 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: FlushMRSOp(f97c5c9bb4c84957857efc76d1c7717f) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":24,"dirs.run_cpu_time_us":146,"dirs.run_wall_time_us":1371,"drs_written":1,"lbm_read_time_us":40,"lbm_reads_lt_1ms":4,"lbm_write_time_us":363,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":62,"threads_started":1,"wal-append.queue_time_us":143}
I20240502 02:00:59.533509 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(f97c5c9bb4c84957857efc76d1c7717f): 4581 bytes on disk
I20240502 02:00:59.533860 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(f97c5c9bb4c84957857efc76d1c7717f) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":44,"lbm_reads_lt_1ms":4}
I20240502 02:00:59.540807 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: FlushMRSOp(f97c5c9bb4c84957857efc76d1c7717f) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":45,"dirs.run_cpu_time_us":155,"dirs.run_wall_time_us":1149,"drs_written":1,"lbm_read_time_us":57,"lbm_reads_lt_1ms":4,"lbm_write_time_us":353,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":104,"threads_started":1,"wal-append.queue_time_us":169}
I20240502 02:00:59.541689 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling UndoDeltaBlockGCOp(1eb15147ff6c4d0b83b12dec736fc576): 4921 bytes on disk
I20240502 02:00:59.542088 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: UndoDeltaBlockGCOp(1eb15147ff6c4d0b83b12dec736fc576) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":68,"lbm_reads_lt_1ms":4}
I20240502 02:00:59.542805 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling UndoDeltaBlockGCOp(f97c5c9bb4c84957857efc76d1c7717f): 4581 bytes on disk
I20240502 02:00:59.543157 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: UndoDeltaBlockGCOp(f97c5c9bb4c84957857efc76d1c7717f) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":55,"lbm_reads_lt_1ms":4}
W20240502 02:00:59.585146 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20240502 02:00:59.620251 10946 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:00:59.627643 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20240502 02:00:59.660076 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling FlushMRSOp(ce85785e670a4264b44797bb27cb4ecb): perf score=0.033391
I20240502 02:00:59.671970 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: FlushMRSOp(ce85785e670a4264b44797bb27cb4ecb) complete. Timing: real 0.012s user 0.010s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":139,"dirs.run_cpu_time_us":183,"dirs.run_wall_time_us":1414,"drs_written":1,"lbm_read_time_us":32,"lbm_reads_lt_1ms":4,"lbm_write_time_us":493,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":147,"threads_started":2,"wal-append.queue_time_us":170}
I20240502 02:00:59.672689 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling UndoDeltaBlockGCOp(ce85785e670a4264b44797bb27cb4ecb): 4834 bytes on disk
I20240502 02:00:59.673027 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: UndoDeltaBlockGCOp(ce85785e670a4264b44797bb27cb4ecb) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":46,"lbm_reads_lt_1ms":4}
I20240502 02:00:59.689914 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(f97c5c9bb4c84957857efc76d1c7717f): perf score=0.033398
I20240502 02:00:59.699867 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(f97c5c9bb4c84957857efc76d1c7717f) complete. Timing: real 0.010s user 0.005s sys 0.004s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":174,"dirs.run_cpu_time_us":170,"dirs.run_wall_time_us":1395,"drs_written":1,"lbm_read_time_us":37,"lbm_reads_lt_1ms":4,"lbm_write_time_us":383,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":114,"threads_started":2,"wal-append.queue_time_us":108}
I20240502 02:00:59.700639 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(f97c5c9bb4c84957857efc76d1c7717f): 4581 bytes on disk
I20240502 02:00:59.700956 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(f97c5c9bb4c84957857efc76d1c7717f) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":43,"lbm_reads_lt_1ms":4}
W20240502 02:00:59.703218 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:00:59.709456 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:00:59.712261 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:00:59.732937 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:00:59.814083 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:00:59.937955 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:00:59.950949 14994 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 99 similar messages]
W20240502 02:00:59.951303 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:00:59.951606 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:01:00.008463 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:01:00.015156 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:01:00.019704 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:00.149524 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 80 similar messages]
I20240502 02:01:00.174741 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling FlushMRSOp(490e37fac0844a08a07a4b281bc33bf9): perf score=0.033359
I20240502 02:01:00.190969 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: FlushMRSOp(490e37fac0844a08a07a4b281bc33bf9) complete. Timing: real 0.016s user 0.016s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":180,"dirs.run_cpu_time_us":378,"dirs.run_wall_time_us":1587,"drs_written":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4,"lbm_write_time_us":461,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":127,"threads_started":2,"wal-append.queue_time_us":130}
I20240502 02:01:00.191752 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling UndoDeltaBlockGCOp(490e37fac0844a08a07a4b281bc33bf9): 4919 bytes on disk
I20240502 02:01:00.192155 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: UndoDeltaBlockGCOp(490e37fac0844a08a07a4b281bc33bf9) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":65,"lbm_reads_lt_1ms":4}
I20240502 02:01:00.203429 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(490e37fac0844a08a07a4b281bc33bf9): perf score=0.033366
I20240502 02:01:00.213349 11068 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 02:01:00.222182 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(490e37fac0844a08a07a4b281bc33bf9) complete. Timing: real 0.019s user 0.014s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":146,"dirs.run_cpu_time_us":168,"dirs.run_wall_time_us":1417,"drs_written":1,"lbm_read_time_us":75,"lbm_reads_lt_1ms":4,"lbm_write_time_us":749,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":123,"threads_started":2,"wal-append.queue_time_us":117}
I20240502 02:01:00.223181 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(ac5ec2c673cb4b988d1099e0442ec63f): perf score=0.033370
I20240502 02:01:00.239152 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(ac5ec2c673cb4b988d1099e0442ec63f) complete. Timing: real 0.016s user 0.013s sys 0.001s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":134,"dirs.run_cpu_time_us":152,"dirs.run_wall_time_us":1383,"drs_written":1,"lbm_read_time_us":78,"lbm_reads_lt_1ms":4,"lbm_write_time_us":586,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":74,"threads_started":1,"wal-append.queue_time_us":145}
I20240502 02:01:00.240422 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(490e37fac0844a08a07a4b281bc33bf9): 4919 bytes on disk
I20240502 02:01:00.240882 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(490e37fac0844a08a07a4b281bc33bf9) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":82,"lbm_reads_lt_1ms":4}
I20240502 02:01:00.242023 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(ac5ec2c673cb4b988d1099e0442ec63f): 4581 bytes on disk
I20240502 02:01:00.242389 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(ac5ec2c673cb4b988d1099e0442ec63f) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":75,"lbm_reads_lt_1ms":4}
W20240502 02:01:00.257742 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 02:01:00.258534 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20240502 02:01:00.275368 10004 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:01:00.275494 10004 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:01:00.276101 10004 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:01:00.276376 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:01:00.276518 15329 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:01:00.277233 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:00.277699 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:00.277760 15396 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:01:00.277899 10004 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:01:00.285056 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:00.286491 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling FlushMRSOp(d1c832c721fb4b15bc6d266a1c295401): perf score=0.033384
I20240502 02:01:00.295534 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling FlushMRSOp(d1c832c721fb4b15bc6d266a1c295401): perf score=0.033392
I20240502 02:01:00.298779 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: FlushMRSOp(d1c832c721fb4b15bc6d266a1c295401) complete. Timing: real 0.012s user 0.008s sys 0.004s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":139,"dirs.run_cpu_time_us":183,"dirs.run_wall_time_us":1223,"drs_written":1,"lbm_read_time_us":48,"lbm_reads_lt_1ms":4,"lbm_write_time_us":474,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":130,"threads_started":2,"wal-append.queue_time_us":142}
I20240502 02:01:00.299656 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(d1c832c721fb4b15bc6d266a1c295401): 4834 bytes on disk
I20240502 02:01:00.300119 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(d1c832c721fb4b15bc6d266a1c295401) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":70,"lbm_reads_lt_1ms":4}
W20240502 02:01:00.306432 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20240502 02:01:00.310757 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: FlushMRSOp(d1c832c721fb4b15bc6d266a1c295401) complete. Timing: real 0.015s user 0.015s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":150,"dirs.run_cpu_time_us":207,"dirs.run_wall_time_us":1169,"drs_written":1,"lbm_read_time_us":35,"lbm_reads_lt_1ms":4,"lbm_write_time_us":877,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"spinlock_wait_cycles":768,"thread_start_us":144,"threads_started":2,"wal-append.queue_time_us":141}
I20240502 02:01:00.311463 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling UndoDeltaBlockGCOp(d1c832c721fb4b15bc6d266a1c295401): 4834 bytes on disk
I20240502 02:01:00.311789 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: UndoDeltaBlockGCOp(d1c832c721fb4b15bc6d266a1c295401) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":44,"lbm_reads_lt_1ms":4}
W20240502 02:01:00.319535 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:00.357487 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 90 similar messages]
W20240502 02:01:00.358121 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:00.380235 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:01:00.403450 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 49 similar messages]
W20240502 02:01:00.431175 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:01:00.468276 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:00.472985 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 02:01:00.493131 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:01:00.595887 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:01:00.606853 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:01:00.629092 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:01:00.656912 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:00.724716 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:00.745285 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(6d5cffb7fcc54118861463e3b1fa4a8d): perf score=0.033364
I20240502 02:01:00.746665 11068 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 02:01:00.758225 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(6d5cffb7fcc54118861463e3b1fa4a8d) complete. Timing: real 0.013s user 0.010s sys 0.001s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":138,"dirs.run_cpu_time_us":196,"dirs.run_wall_time_us":1401,"drs_written":1,"lbm_read_time_us":64,"lbm_reads_lt_1ms":4,"lbm_write_time_us":438,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":114,"threads_started":2,"wal-append.queue_time_us":125}
I20240502 02:01:00.759244 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(6d5cffb7fcc54118861463e3b1fa4a8d): 4581 bytes on disk
I20240502 02:01:00.759600 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(6d5cffb7fcc54118861463e3b1fa4a8d) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":45,"lbm_reads_lt_1ms":4}
I20240502 02:01:00.802181 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling FlushMRSOp(81c8fe21ec7545e79450532b43a61ee7): perf score=0.033379
I20240502 02:01:00.813402 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: FlushMRSOp(81c8fe21ec7545e79450532b43a61ee7) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":185,"dirs.run_cpu_time_us":208,"dirs.run_wall_time_us":1346,"drs_written":1,"lbm_read_time_us":35,"lbm_reads_lt_1ms":4,"lbm_write_time_us":379,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":149,"threads_started":2,"wal-append.queue_time_us":121}
I20240502 02:01:00.813889 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling FlushMRSOp(8f2fb504a49c44edac92fd935555c278): perf score=0.033381
I20240502 02:01:00.814294 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(81c8fe21ec7545e79450532b43a61ee7): 4921 bytes on disk
I20240502 02:01:00.814689 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(81c8fe21ec7545e79450532b43a61ee7) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":76,"lbm_reads_lt_1ms":4}
I20240502 02:01:00.815337 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling FlushMRSOp(8f2fb504a49c44edac92fd935555c278): perf score=0.033381
W20240502 02:01:00.827440 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:00.829988 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: FlushMRSOp(8f2fb504a49c44edac92fd935555c278) complete. Timing: real 0.016s user 0.014s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":182,"dirs.run_cpu_time_us":184,"dirs.run_wall_time_us":1189,"drs_written":1,"lbm_read_time_us":67,"lbm_reads_lt_1ms":4,"lbm_write_time_us":572,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":197,"threads_started":2,"wal-append.queue_time_us":178}
I20240502 02:01:00.830773 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling UndoDeltaBlockGCOp(8f2fb504a49c44edac92fd935555c278): 4834 bytes on disk
I20240502 02:01:00.831099 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: UndoDeltaBlockGCOp(8f2fb504a49c44edac92fd935555c278) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
I20240502 02:01:00.831789 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: FlushMRSOp(8f2fb504a49c44edac92fd935555c278) complete. Timing: real 0.016s user 0.015s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":38,"dirs.run_cpu_time_us":179,"dirs.run_wall_time_us":1671,"drs_written":1,"lbm_read_time_us":52,"lbm_reads_lt_1ms":4,"lbm_write_time_us":515,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":69,"threads_started":1,"wal-append.queue_time_us":138}
I20240502 02:01:00.832502 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(8f2fb504a49c44edac92fd935555c278): 4834 bytes on disk
I20240502 02:01:00.832799 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(8f2fb504a49c44edac92fd935555c278) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":44,"lbm_reads_lt_1ms":4}
W20240502 02:01:00.864956 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:00.884568 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:01:00.905830 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:01:00.938439 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:01:01.173579 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:01.195318 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling FlushMRSOp(30620f4774b8414eb4e27bd5ecb6ecee): perf score=0.033360
I20240502 02:01:01.207552 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: FlushMRSOp(30620f4774b8414eb4e27bd5ecb6ecee) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":167,"dirs.run_cpu_time_us":222,"dirs.run_wall_time_us":1573,"drs_written":1,"lbm_read_time_us":42,"lbm_reads_lt_1ms":4,"lbm_write_time_us":478,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":178,"threads_started":2,"wal-append.queue_time_us":171}
I20240502 02:01:01.208302 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling UndoDeltaBlockGCOp(30620f4774b8414eb4e27bd5ecb6ecee): 4834 bytes on disk
I20240502 02:01:01.208638 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: UndoDeltaBlockGCOp(30620f4774b8414eb4e27bd5ecb6ecee) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":48,"lbm_reads_lt_1ms":4}
W20240502 02:01:01.239786 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:01.242946 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:01.255769 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20240502 02:01:01.311637 11068 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 02:01:01.332947 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling FlushMRSOp(77ae087802144c45924c1b79f561afe8): perf score=0.033397
I20240502 02:01:01.334640 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling FlushMRSOp(30620f4774b8414eb4e27bd5ecb6ecee): perf score=0.033396
I20240502 02:01:01.348294 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: FlushMRSOp(77ae087802144c45924c1b79f561afe8) complete. Timing: real 0.015s user 0.014s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":157,"dirs.run_cpu_time_us":184,"dirs.run_wall_time_us":1585,"drs_written":1,"lbm_read_time_us":57,"lbm_reads_lt_1ms":4,"lbm_write_time_us":470,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":131,"threads_started":2,"wal-append.queue_time_us":129}
I20240502 02:01:01.349190 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling FlushMRSOp(e00d1ed5cde044cb9bdebf787590261a): perf score=0.033401
I20240502 02:01:01.349800 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: FlushMRSOp(30620f4774b8414eb4e27bd5ecb6ecee) complete. Timing: real 0.015s user 0.013s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":138,"dirs.run_cpu_time_us":95,"dirs.run_wall_time_us":1139,"drs_written":1,"lbm_read_time_us":43,"lbm_reads_lt_1ms":4,"lbm_write_time_us":504,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":115,"threads_started":2,"wal-append.queue_time_us":117}
I20240502 02:01:01.350767 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(30620f4774b8414eb4e27bd5ecb6ecee): 4834 bytes on disk
I20240502 02:01:01.351195 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(30620f4774b8414eb4e27bd5ecb6ecee) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":60,"lbm_reads_lt_1ms":4}
I20240502 02:01:01.360795 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: FlushMRSOp(e00d1ed5cde044cb9bdebf787590261a) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":219,"dirs.run_cpu_time_us":176,"dirs.run_wall_time_us":1183,"drs_written":1,"lbm_read_time_us":41,"lbm_reads_lt_1ms":4,"lbm_write_time_us":517,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":71,"threads_started":1,"wal-append.queue_time_us":145}
I20240502 02:01:01.361500 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling UndoDeltaBlockGCOp(77ae087802144c45924c1b79f561afe8): 4921 bytes on disk
I20240502 02:01:01.361815 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: UndoDeltaBlockGCOp(77ae087802144c45924c1b79f561afe8) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":48,"lbm_reads_lt_1ms":4}
I20240502 02:01:01.362390 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling UndoDeltaBlockGCOp(e00d1ed5cde044cb9bdebf787590261a): 4581 bytes on disk
I20240502 02:01:01.362709 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: UndoDeltaBlockGCOp(e00d1ed5cde044cb9bdebf787590261a) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":46,"lbm_reads_lt_1ms":4}
W20240502 02:01:01.367431 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:01:01.380708 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:01.396248 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20240502 02:01:01.459760 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling FlushMRSOp(cd00409d47954ee2b7555e7ff09a6e28): perf score=0.033343
W20240502 02:01:01.463430 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20240502 02:01:01.471629 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: FlushMRSOp(cd00409d47954ee2b7555e7ff09a6e28) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":51,"dirs.run_cpu_time_us":222,"dirs.run_wall_time_us":1449,"drs_written":1,"lbm_read_time_us":32,"lbm_reads_lt_1ms":4,"lbm_write_time_us":408,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":80,"threads_started":1,"wal-append.queue_time_us":150}
I20240502 02:01:01.472409 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling UndoDeltaBlockGCOp(cd00409d47954ee2b7555e7ff09a6e28): 4921 bytes on disk
I20240502 02:01:01.472750 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: UndoDeltaBlockGCOp(cd00409d47954ee2b7555e7ff09a6e28) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":46,"lbm_reads_lt_1ms":4}
I20240502 02:01:01.473392 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling FlushMRSOp(9c2892024ce942e1bf1834c1b9ad2049): perf score=0.033344
I20240502 02:01:01.483456 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: FlushMRSOp(9c2892024ce942e1bf1834c1b9ad2049) complete. Timing: real 0.010s user 0.009s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":46,"dirs.run_cpu_time_us":182,"dirs.run_wall_time_us":1250,"drs_written":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4,"lbm_write_time_us":373,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":74,"threads_started":1,"wal-append.queue_time_us":140}
I20240502 02:01:01.484068 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling UndoDeltaBlockGCOp(9c2892024ce942e1bf1834c1b9ad2049): 4834 bytes on disk
I20240502 02:01:01.484395 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: UndoDeltaBlockGCOp(9c2892024ce942e1bf1834c1b9ad2049) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
W20240502 02:01:01.518894 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:01.588711 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:01.593844 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:01.602795 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling FlushMRSOp(d24986c2ebab46f086e3f1f5a53af26b): perf score=0.033385
I20240502 02:01:01.613438 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: FlushMRSOp(d24986c2ebab46f086e3f1f5a53af26b) complete. Timing: real 0.010s user 0.009s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":57,"dirs.run_cpu_time_us":131,"dirs.run_wall_time_us":1410,"drs_written":1,"lbm_read_time_us":33,"lbm_reads_lt_1ms":4,"lbm_write_time_us":378,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":81,"threads_started":1,"wal-append.queue_time_us":189}
I20240502 02:01:01.613958 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling FlushMRSOp(9c2892024ce942e1bf1834c1b9ad2049): perf score=0.033383
I20240502 02:01:01.614305 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(d24986c2ebab46f086e3f1f5a53af26b): 4581 bytes on disk
I20240502 02:01:01.614718 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(d24986c2ebab46f086e3f1f5a53af26b) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":63,"lbm_reads_lt_1ms":4}
I20240502 02:01:01.615427 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling FlushMRSOp(cd00409d47954ee2b7555e7ff09a6e28): perf score=0.033387
W20240502 02:01:01.623373 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:01.623735 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20240502 02:01:01.627447 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: FlushMRSOp(9c2892024ce942e1bf1834c1b9ad2049) complete. Timing: real 0.013s user 0.012s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":35,"dirs.run_cpu_time_us":117,"dirs.run_wall_time_us":1094,"drs_written":1,"lbm_read_time_us":62,"lbm_reads_lt_1ms":4,"lbm_write_time_us":500,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":65,"threads_started":1,"wal-append.queue_time_us":148}
I20240502 02:01:01.628293 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling UndoDeltaBlockGCOp(9c2892024ce942e1bf1834c1b9ad2049): 4834 bytes on disk
I20240502 02:01:01.628597 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: UndoDeltaBlockGCOp(9c2892024ce942e1bf1834c1b9ad2049) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
I20240502 02:01:01.628996 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: FlushMRSOp(cd00409d47954ee2b7555e7ff09a6e28) complete. Timing: real 0.013s user 0.012s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":40,"dirs.run_cpu_time_us":168,"dirs.run_wall_time_us":1321,"drs_written":1,"lbm_read_time_us":35,"lbm_reads_lt_1ms":4,"lbm_write_time_us":347,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":84,"threads_started":1,"wal-append.queue_time_us":157}
I20240502 02:01:01.629637 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(cd00409d47954ee2b7555e7ff09a6e28): 4921 bytes on disk
I20240502 02:01:01.629961 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(cd00409d47954ee2b7555e7ff09a6e28) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
W20240502 02:01:01.655016 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:01.676797 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20240502 02:01:01.769366 11068 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:01.809912 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:01.817893 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:01.879981 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling FlushMRSOp(6c4e85e7050b4dc8a4ce6bba818854a2): perf score=0.033356
I20240502 02:01:01.881943 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling FlushMRSOp(856f623374824d94913db0cb9faa25d1): perf score=0.033357
I20240502 02:01:01.892493 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: FlushMRSOp(6c4e85e7050b4dc8a4ce6bba818854a2) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":53,"dirs.run_cpu_time_us":170,"dirs.run_wall_time_us":1378,"drs_written":1,"lbm_read_time_us":67,"lbm_reads_lt_1ms":4,"lbm_write_time_us":440,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":78,"threads_started":1,"wal-append.queue_time_us":163}
I20240502 02:01:01.893256 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling UndoDeltaBlockGCOp(6c4e85e7050b4dc8a4ce6bba818854a2): 4834 bytes on disk
I20240502 02:01:01.893596 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: UndoDeltaBlockGCOp(6c4e85e7050b4dc8a4ce6bba818854a2) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4}
I20240502 02:01:01.894424 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: FlushMRSOp(856f623374824d94913db0cb9faa25d1) complete. Timing: real 0.012s user 0.007s sys 0.004s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":48,"dirs.run_cpu_time_us":182,"dirs.run_wall_time_us":1011,"drs_written":1,"lbm_read_time_us":93,"lbm_reads_lt_1ms":4,"lbm_write_time_us":515,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":72,"threads_started":1,"wal-append.queue_time_us":162}
I20240502 02:01:01.895179 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(856f623374824d94913db0cb9faa25d1): 4921 bytes on disk
I20240502 02:01:01.895509 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(856f623374824d94913db0cb9faa25d1) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4}
I20240502 02:01:01.896095 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling FlushMRSOp(a901eb3063ea4911a822552db01d365d): perf score=0.033358
I20240502 02:01:01.907786 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: FlushMRSOp(a901eb3063ea4911a822552db01d365d) complete. Timing: real 0.012s user 0.010s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":46,"dirs.run_cpu_time_us":175,"dirs.run_wall_time_us":1192,"drs_written":1,"lbm_read_time_us":35,"lbm_reads_lt_1ms":4,"lbm_write_time_us":363,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":63,"threads_started":1,"wal-append.queue_time_us":158}
I20240502 02:01:01.908509 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(a901eb3063ea4911a822552db01d365d): 4581 bytes on disk
I20240502 02:01:01.908882 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(a901eb3063ea4911a822552db01d365d) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":62,"lbm_reads_lt_1ms":4}
W20240502 02:01:01.937228 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20240502 02:01:01.986178 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling FlushMRSOp(6c4e85e7050b4dc8a4ce6bba818854a2): perf score=0.033386
I20240502 02:01:01.998620 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: FlushMRSOp(6c4e85e7050b4dc8a4ce6bba818854a2) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":189,"dirs.run_cpu_time_us":171,"dirs.run_wall_time_us":1332,"drs_written":1,"lbm_read_time_us":40,"lbm_reads_lt_1ms":4,"lbm_write_time_us":416,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":209,"threads_started":2,"wal-append.queue_time_us":184}
I20240502 02:01:01.999549 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling UndoDeltaBlockGCOp(6c4e85e7050b4dc8a4ce6bba818854a2): 4834 bytes on disk
I20240502 02:01:01.999893 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: UndoDeltaBlockGCOp(6c4e85e7050b4dc8a4ce6bba818854a2) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":48,"lbm_reads_lt_1ms":4}
W20240502 02:01:02.045449 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:02.048422 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:02.051908 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:02.095566 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:02.105685 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:02.147619 11454 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:02.256743 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:02.274889 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:02.325598 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:02.327402 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:02.364496 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:01:02.416064 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:02.493813 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:02.518312 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:02.668924 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:02.705334 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:02.725171 11454 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:02.777957 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:01:02.845654 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:02.871181 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:02.961308 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 02:01:03.005823 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:03.033437 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:03.043082 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:03.058221 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:03.071986 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:03.081164 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:03.129531 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:03.140064 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:03.167593 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:03.177084 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:03.298832 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20240502 02:01:03.302340 11454 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:03.423740 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:03.429643 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:03.429620 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:03.431937 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:03.434558 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:03.546049 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
I20240502 02:01:03.745250 11850 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:03.751116 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:03.777523 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:03.790755 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:03.924430 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:03.940142 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:03.955593 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:03.969319 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:04.090878 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
I20240502 02:01:04.143527 11454 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:04.257855 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:04.315754 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:04.343973 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:04.365906 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:04.449790 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:04.454425 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:04.472378 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:04.514842 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:04.542356 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:04.559026 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:04.585889 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:04.700775 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:04.707752 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
I20240502 02:01:04.731422 11454 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:04.741005 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:04.779709 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:04.853243 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:04.859968 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:04.902205 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:04.909700 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:04.932598 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:04.955915 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 99 similar messages]
W20240502 02:01:05.112190 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
I20240502 02:01:05.182202 12142 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:05.255530 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 84 similar messages]
W20240502 02:01:05.292819 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:05.337198 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:05.353120 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 68 similar messages]
W20240502 02:01:05.417181 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:05.423038 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 48 similar messages]
W20240502 02:01:05.423614 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 02:01:05.423928 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 88 similar messages]
W20240502 02:01:05.424453 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:05.469033 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:05.472115 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:05.473207 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:05.484423 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:05.666545 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:05.687301 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:05.743575 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
I20240502 02:01:05.761248 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling FlushMRSOp(d24986c2ebab46f086e3f1f5a53af26b): perf score=0.033364
I20240502 02:01:05.783636 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: FlushMRSOp(d24986c2ebab46f086e3f1f5a53af26b) complete. Timing: real 0.022s user 0.021s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"compiler_manager_pool.queue_time_us":192,"dirs.queue_time_us":164,"dirs.run_cpu_time_us":193,"dirs.run_wall_time_us":1066,"drs_written":1,"lbm_read_time_us":48,"lbm_reads_lt_1ms":4,"lbm_write_time_us":376,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":255,"threads_started":3,"wal-append.queue_time_us":140}
I20240502 02:01:05.784897 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling UndoDeltaBlockGCOp(d24986c2ebab46f086e3f1f5a53af26b): 4581 bytes on disk
I20240502 02:01:05.785446 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: UndoDeltaBlockGCOp(d24986c2ebab46f086e3f1f5a53af26b) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":68,"lbm_reads_lt_1ms":4}
I20240502 02:01:05.786093 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling FlushMRSOp(9c2892024ce942e1bf1834c1b9ad2049): perf score=0.033369
I20240502 02:01:05.798686 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: FlushMRSOp(9c2892024ce942e1bf1834c1b9ad2049) complete. Timing: real 0.012s user 0.012s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":58,"dirs.run_cpu_time_us":181,"dirs.run_wall_time_us":1539,"drs_written":1,"lbm_read_time_us":33,"lbm_reads_lt_1ms":4,"lbm_write_time_us":348,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":56,"threads_started":1,"wal-append.queue_time_us":125}
I20240502 02:01:05.799456 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling FlushMRSOp(8ddc239a297a46e48e069b985cb7b350): perf score=0.033349
I20240502 02:01:05.805397 12142 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 02:01:05.813522 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: FlushMRSOp(8ddc239a297a46e48e069b985cb7b350) complete. Timing: real 0.014s user 0.012s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":51,"dirs.run_cpu_time_us":110,"dirs.run_wall_time_us":1331,"drs_written":1,"lbm_read_time_us":61,"lbm_reads_lt_1ms":4,"lbm_write_time_us":416,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":88,"threads_started":1,"wal-append.queue_time_us":168}
I20240502 02:01:05.814429 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling UndoDeltaBlockGCOp(8ddc239a297a46e48e069b985cb7b350): 4834 bytes on disk
I20240502 02:01:05.815477 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: UndoDeltaBlockGCOp(8ddc239a297a46e48e069b985cb7b350) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4}
I20240502 02:01:05.816203 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling UndoDeltaBlockGCOp(9c2892024ce942e1bf1834c1b9ad2049): 4834 bytes on disk
I20240502 02:01:05.816500 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: UndoDeltaBlockGCOp(9c2892024ce942e1bf1834c1b9ad2049) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":52,"lbm_reads_lt_1ms":4}
W20240502 02:01:05.842217 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:05.852754 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:05.896613 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:05.909811 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:05.984568 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:06.001670 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:06.020323 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:06.029753 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:06.036069 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
I20240502 02:01:06.067917 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling FlushMRSOp(856f623374824d94913db0cb9faa25d1): perf score=0.033363
I20240502 02:01:06.083078 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: FlushMRSOp(856f623374824d94913db0cb9faa25d1) complete. Timing: real 0.015s user 0.014s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":41,"dirs.run_cpu_time_us":194,"dirs.run_wall_time_us":1386,"drs_written":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4,"lbm_write_time_us":665,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":62,"threads_started":1,"wal-append.queue_time_us":133}
I20240502 02:01:06.083990 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling UndoDeltaBlockGCOp(856f623374824d94913db0cb9faa25d1): 4921 bytes on disk
I20240502 02:01:06.084429 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: UndoDeltaBlockGCOp(856f623374824d94913db0cb9faa25d1) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4}
I20240502 02:01:06.085112 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling FlushMRSOp(e2f4ca025890492880608c7975650969): perf score=0.033362
W20240502 02:01:06.092309 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
I20240502 02:01:06.097234 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: FlushMRSOp(e2f4ca025890492880608c7975650969) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":42,"dirs.run_cpu_time_us":129,"dirs.run_wall_time_us":1163,"drs_written":1,"lbm_read_time_us":34,"lbm_reads_lt_1ms":4,"lbm_write_time_us":495,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":76,"threads_started":1,"wal-append.queue_time_us":139}
I20240502 02:01:06.098064 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling UndoDeltaBlockGCOp(e2f4ca025890492880608c7975650969): 4581 bytes on disk
I20240502 02:01:06.098449 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: UndoDeltaBlockGCOp(e2f4ca025890492880608c7975650969) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4}
I20240502 02:01:06.099169 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling FlushMRSOp(e00d1ed5cde044cb9bdebf787590261a): perf score=0.033365
I20240502 02:01:06.118203 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: FlushMRSOp(e00d1ed5cde044cb9bdebf787590261a) complete. Timing: real 0.015s user 0.009s sys 0.003s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":54,"dirs.run_cpu_time_us":192,"dirs.run_wall_time_us":1147,"drs_written":1,"lbm_read_time_us":36,"lbm_reads_lt_1ms":4,"lbm_write_time_us":362,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":79,"threads_started":1,"wal-append.queue_time_us":146}
I20240502 02:01:06.119019 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling UndoDeltaBlockGCOp(e00d1ed5cde044cb9bdebf787590261a): 4581 bytes on disk
I20240502 02:01:06.119393 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: UndoDeltaBlockGCOp(e00d1ed5cde044cb9bdebf787590261a) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":74,"lbm_reads_lt_1ms":4}
I20240502 02:01:06.120148 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling FlushMRSOp(6d5cffb7fcc54118861463e3b1fa4a8d): perf score=0.033356
I20240502 02:01:06.132548 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: FlushMRSOp(6d5cffb7fcc54118861463e3b1fa4a8d) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":33,"dirs.run_cpu_time_us":154,"dirs.run_wall_time_us":1152,"drs_written":1,"lbm_read_time_us":34,"lbm_reads_lt_1ms":4,"lbm_write_time_us":448,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":78,"threads_started":1,"wal-append.queue_time_us":142}
I20240502 02:01:06.133185 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling UndoDeltaBlockGCOp(6d5cffb7fcc54118861463e3b1fa4a8d): 4581 bytes on disk
I20240502 02:01:06.133495 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: UndoDeltaBlockGCOp(6d5cffb7fcc54118861463e3b1fa4a8d) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":45,"lbm_reads_lt_1ms":4}
W20240502 02:01:06.207755 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
I20240502 02:01:06.208953 12355 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:06.220957 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:06.297531 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:06.384770 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling FlushMRSOp(fcfc8dc4b90a4a6aa7e4035bbfdb4532): perf score=0.033376
I20240502 02:01:06.396728 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: FlushMRSOp(fcfc8dc4b90a4a6aa7e4035bbfdb4532) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":53,"dirs.run_cpu_time_us":201,"dirs.run_wall_time_us":1495,"drs_written":1,"lbm_read_time_us":45,"lbm_reads_lt_1ms":4,"lbm_write_time_us":470,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":85,"threads_started":1,"wal-append.queue_time_us":148}
I20240502 02:01:06.397675 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling FlushMRSOp(77ae087802144c45924c1b79f561afe8): perf score=0.033336
I20240502 02:01:06.409143 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: FlushMRSOp(77ae087802144c45924c1b79f561afe8) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":65,"dirs.run_cpu_time_us":212,"dirs.run_wall_time_us":1317,"drs_written":1,"lbm_read_time_us":33,"lbm_reads_lt_1ms":4,"lbm_write_time_us":432,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":100,"threads_started":1,"wal-append.queue_time_us":169}
I20240502 02:01:06.409828 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling UndoDeltaBlockGCOp(77ae087802144c45924c1b79f561afe8): 4921 bytes on disk
I20240502 02:01:06.410205 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: UndoDeltaBlockGCOp(77ae087802144c45924c1b79f561afe8) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":62,"lbm_reads_lt_1ms":4}
I20240502 02:01:06.410974 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling UndoDeltaBlockGCOp(fcfc8dc4b90a4a6aa7e4035bbfdb4532): 4834 bytes on disk
W20240502 02:01:06.411348 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:06.411334 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: UndoDeltaBlockGCOp(fcfc8dc4b90a4a6aa7e4035bbfdb4532) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":59,"lbm_reads_lt_1ms":4}
W20240502 02:01:06.486166 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:06.498157 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:06.592233 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
I20240502 02:01:06.622157 12355 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:06.627247 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:06.811606 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:06.820619 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:06.844933 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:06.845953 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:06.885495 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:06.899852 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:06.941365 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:07.034113 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:07.054533 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
I20240502 02:01:07.105191 12355 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:07.154776 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:07.177839 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:07.186991 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:07.196249 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:07.201364 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:07.231827 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:07.263736 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:07.365108 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:07.414455 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling FlushMRSOp(6c4e85e7050b4dc8a4ce6bba818854a2): perf score=0.033399
I20240502 02:01:07.426272 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: FlushMRSOp(6c4e85e7050b4dc8a4ce6bba818854a2) complete. Timing: real 0.012s user 0.010s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":139,"dirs.run_cpu_time_us":176,"dirs.run_wall_time_us":1291,"drs_written":1,"lbm_read_time_us":34,"lbm_reads_lt_1ms":4,"lbm_write_time_us":453,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":114,"threads_started":2,"wal-append.queue_time_us":108}
I20240502 02:01:07.427119 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling UndoDeltaBlockGCOp(6c4e85e7050b4dc8a4ce6bba818854a2): 4834 bytes on disk
I20240502 02:01:07.427465 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: UndoDeltaBlockGCOp(6c4e85e7050b4dc8a4ce6bba818854a2) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":42,"lbm_reads_lt_1ms":4}
W20240502 02:01:07.471750 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:07.480077 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:07.493507 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:07.507555 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:07.535898 12142 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:07.700033 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:07.708480 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:07.807538 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:07.865803 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:07.891793 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:07.979499 12355 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:08.021241 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 02:01:08.025357 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:08.027529 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:08.040525 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:08.063035 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:08.128875 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:08.193305 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:08.282310 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:08.310395 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:08.427443 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:08.471101 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:08.498203 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:08.499382 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:08.501271 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:08.518276 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:08.565752 12355 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:08.573146 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
I20240502 02:01:08.680837 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling FlushMRSOp(f97c5c9bb4c84957857efc76d1c7717f): perf score=0.033367
I20240502 02:01:08.691510 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: FlushMRSOp(f97c5c9bb4c84957857efc76d1c7717f) complete. Timing: real 0.010s user 0.009s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":139,"dirs.run_cpu_time_us":131,"dirs.run_wall_time_us":1369,"drs_written":1,"lbm_read_time_us":40,"lbm_reads_lt_1ms":4,"lbm_write_time_us":359,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":127,"threads_started":2,"wal-append.queue_time_us":121}
I20240502 02:01:08.692291 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling UndoDeltaBlockGCOp(f97c5c9bb4c84957857efc76d1c7717f): 4581 bytes on disk
I20240502 02:01:08.692593 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: UndoDeltaBlockGCOp(f97c5c9bb4c84957857efc76d1c7717f) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":39,"lbm_reads_lt_1ms":4}
W20240502 02:01:08.791585 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:08.830183 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:08.873178 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:08.972347 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:08.990376 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:08.991291 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:09.003091 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
I20240502 02:01:09.018759 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling FlushMRSOp(81c8fe21ec7545e79450532b43a61ee7): perf score=0.033401
W20240502 02:01:09.025363 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:09.031366 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: FlushMRSOp(81c8fe21ec7545e79450532b43a61ee7) complete. Timing: real 0.012s user 0.012s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":188,"dirs.run_cpu_time_us":196,"dirs.run_wall_time_us":1466,"drs_written":1,"lbm_read_time_us":37,"lbm_reads_lt_1ms":4,"lbm_write_time_us":397,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":150,"threads_started":2,"wal-append.queue_time_us":123}
I20240502 02:01:09.032056 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling UndoDeltaBlockGCOp(81c8fe21ec7545e79450532b43a61ee7): 4921 bytes on disk
I20240502 02:01:09.032449 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: UndoDeltaBlockGCOp(81c8fe21ec7545e79450532b43a61ee7) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":50,"lbm_reads_lt_1ms":4}
W20240502 02:01:09.117877 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:09.172940 13094 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 02:01:09.179716 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling FlushMRSOp(1eb15147ff6c4d0b83b12dec736fc576): perf score=0.033367
I20240502 02:01:09.193564 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: FlushMRSOp(1eb15147ff6c4d0b83b12dec736fc576) complete. Timing: real 0.014s user 0.012s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":152,"dirs.run_cpu_time_us":148,"dirs.run_wall_time_us":1262,"drs_written":1,"lbm_read_time_us":37,"lbm_reads_lt_1ms":4,"lbm_write_time_us":460,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":125,"threads_started":2,"wal-append.queue_time_us":110}
I20240502 02:01:09.194295 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(1eb15147ff6c4d0b83b12dec736fc576): 4921 bytes on disk
I20240502 02:01:09.194687 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(1eb15147ff6c4d0b83b12dec736fc576) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4}
W20240502 02:01:09.240674 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:09.275600 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
I20240502 02:01:09.285780 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(cd00409d47954ee2b7555e7ff09a6e28): perf score=0.033380
I20240502 02:01:09.297230 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(cd00409d47954ee2b7555e7ff09a6e28) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":156,"dirs.run_cpu_time_us":188,"dirs.run_wall_time_us":1237,"drs_written":1,"lbm_read_time_us":34,"lbm_reads_lt_1ms":4,"lbm_write_time_us":465,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":122,"threads_started":2,"wal-append.queue_time_us":113}
I20240502 02:01:09.298244 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(77ae087802144c45924c1b79f561afe8): perf score=0.033383
W20240502 02:01:09.299176 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
I20240502 02:01:09.308996 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(77ae087802144c45924c1b79f561afe8) complete. Timing: real 0.011s user 0.009s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":39,"dirs.run_cpu_time_us":135,"dirs.run_wall_time_us":1102,"drs_written":1,"lbm_read_time_us":35,"lbm_reads_lt_1ms":4,"lbm_write_time_us":437,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":62,"threads_started":1,"wal-append.queue_time_us":130}
I20240502 02:01:09.309939 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(77ae087802144c45924c1b79f561afe8): 4921 bytes on disk
I20240502 02:01:09.310364 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(77ae087802144c45924c1b79f561afe8) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":57,"lbm_reads_lt_1ms":4}
I20240502 02:01:09.311081 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(cd00409d47954ee2b7555e7ff09a6e28): 4921 bytes on disk
I20240502 02:01:09.311391 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(cd00409d47954ee2b7555e7ff09a6e28) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":43,"lbm_reads_lt_1ms":4}
W20240502 02:01:09.331604 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:09.352026 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:09.445996 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:09.448482 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:09.482998 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:09.517382 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:09.556588 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:09.557720 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:09.691627 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:09.695597 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling FlushMRSOp(3f938c5af51a43c9a23b753c4991e41d): perf score=0.033388
I20240502 02:01:09.708894 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: FlushMRSOp(3f938c5af51a43c9a23b753c4991e41d) complete. Timing: real 0.013s user 0.012s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":165,"dirs.run_cpu_time_us":185,"dirs.run_wall_time_us":1521,"drs_written":1,"lbm_read_time_us":52,"lbm_reads_lt_1ms":4,"lbm_write_time_us":434,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":157,"threads_started":2,"wal-append.queue_time_us":115}
I20240502 02:01:09.709578 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling UndoDeltaBlockGCOp(3f938c5af51a43c9a23b753c4991e41d): 4834 bytes on disk
I20240502 02:01:09.709926 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: UndoDeltaBlockGCOp(3f938c5af51a43c9a23b753c4991e41d) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":61,"lbm_reads_lt_1ms":4}
W20240502 02:01:09.729142 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:09.765252 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
I20240502 02:01:09.774791 12355 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:09.781694 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
I20240502 02:01:09.813730 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(a901eb3063ea4911a822552db01d365d): perf score=0.033402
I20240502 02:01:09.824656 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(a901eb3063ea4911a822552db01d365d) complete. Timing: real 0.011s user 0.009s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":168,"dirs.run_cpu_time_us":142,"dirs.run_wall_time_us":1339,"drs_written":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4,"lbm_write_time_us":435,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":149,"threads_started":2,"wal-append.queue_time_us":140}
I20240502 02:01:09.825517 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(a901eb3063ea4911a822552db01d365d): 4581 bytes on disk
I20240502 02:01:09.825878 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(a901eb3063ea4911a822552db01d365d) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":46,"lbm_reads_lt_1ms":4}
I20240502 02:01:09.826486 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(d24986c2ebab46f086e3f1f5a53af26b): perf score=0.033335
I20240502 02:01:09.836596 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(d24986c2ebab46f086e3f1f5a53af26b) complete. Timing: real 0.010s user 0.009s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":34,"dirs.run_cpu_time_us":165,"dirs.run_wall_time_us":1230,"drs_written":1,"lbm_read_time_us":33,"lbm_reads_lt_1ms":4,"lbm_write_time_us":343,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":97,"threads_started":1,"wal-append.queue_time_us":161}
I20240502 02:01:09.837294 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(d24986c2ebab46f086e3f1f5a53af26b): 4581 bytes on disk
I20240502 02:01:09.837575 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(d24986c2ebab46f086e3f1f5a53af26b) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":43,"lbm_reads_lt_1ms":4}
W20240502 02:01:09.886829 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:09.910677 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:09.916430 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling FlushMRSOp(21e2e55d1a0d4b6eb18b4ade9468607f): perf score=0.033395
I20240502 02:01:09.929903 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: FlushMRSOp(21e2e55d1a0d4b6eb18b4ade9468607f) complete. Timing: real 0.013s user 0.012s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":156,"dirs.run_cpu_time_us":175,"dirs.run_wall_time_us":1303,"drs_written":1,"lbm_read_time_us":65,"lbm_reads_lt_1ms":4,"lbm_write_time_us":550,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"spinlock_wait_cycles":3328,"thread_start_us":155,"threads_started":2,"wal-append.queue_time_us":148}
I20240502 02:01:09.930914 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling UndoDeltaBlockGCOp(21e2e55d1a0d4b6eb18b4ade9468607f): 4921 bytes on disk
I20240502 02:01:09.931730 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: UndoDeltaBlockGCOp(21e2e55d1a0d4b6eb18b4ade9468607f) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":91,"lbm_reads_lt_1ms":4}
W20240502 02:01:09.973336 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:10.031280 14992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 99 similar messages]
I20240502 02:01:10.095985 13217 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:01:10.096138 13217 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:01:10.096696 13217 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:01:10.097015 15464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 02:01:10.097160 15464 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 02:01:10.097680 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:10.097779 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:10.097872 15266 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:01:10.098004 13217 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:01:10.132241 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:10.168721 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:10.211380 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:10.212261 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling FlushMRSOp(091d27d51acc4db0a1c7d20deeff5c10): perf score=0.033385
W20240502 02:01:10.224018 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
I20240502 02:01:10.226344 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: FlushMRSOp(091d27d51acc4db0a1c7d20deeff5c10) complete. Timing: real 0.014s user 0.012s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":826,"dirs.run_cpu_time_us":179,"dirs.run_wall_time_us":3473,"drs_written":1,"lbm_read_time_us":108,"lbm_reads_lt_1ms":4,"lbm_write_time_us":394,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":137,"threads_started":2,"wal-append.queue_time_us":126}
I20240502 02:01:10.227044 16888 maintenance_manager.cc:419] P 84987ad9e4a54d96af8569e4aaf0ed64: Scheduling UndoDeltaBlockGCOp(091d27d51acc4db0a1c7d20deeff5c10): 4581 bytes on disk
I20240502 02:01:10.227360 16775 maintenance_manager.cc:643] P 84987ad9e4a54d96af8569e4aaf0ed64: UndoDeltaBlockGCOp(091d27d51acc4db0a1c7d20deeff5c10) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4}
I20240502 02:01:10.247184 13094 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:10.278398 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 02:01:10.318442 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
I20240502 02:01:10.339665 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling FlushMRSOp(6db29f2516e1486782be0312c9bcac8b): perf score=0.033364
I20240502 02:01:10.351994 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: FlushMRSOp(6db29f2516e1486782be0312c9bcac8b) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":182,"dirs.run_cpu_time_us":224,"dirs.run_wall_time_us":1564,"drs_written":1,"lbm_read_time_us":40,"lbm_reads_lt_1ms":4,"lbm_write_time_us":481,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":163,"threads_started":2,"wal-append.queue_time_us":136}
I20240502 02:01:10.352867 15517 maintenance_manager.cc:419] P 7b66a8940fe54e21bec1b2db25b2fed5: Scheduling UndoDeltaBlockGCOp(6db29f2516e1486782be0312c9bcac8b): 4921 bytes on disk
I20240502 02:01:10.353221 15400 maintenance_manager.cc:643] P 7b66a8940fe54e21bec1b2db25b2fed5: UndoDeltaBlockGCOp(6db29f2516e1486782be0312c9bcac8b) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4}
W20240502 02:01:10.357782 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:10.389616 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:10.430462 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:10.439193 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 93 similar messages]
W20240502 02:01:10.448988 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 73 similar messages]
W20240502 02:01:10.449743 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:10.562319 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:10.575762 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 49 similar messages]
W20240502 02:01:10.576323 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:10.607510 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
I20240502 02:01:10.680197 13094 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:10.692021 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:10.707054 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 02:01:10.764534 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:10.781965 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:10.834332 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:10.845705 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:10.890197 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:10.925678 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:10.974773 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:11.080931 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:11.163494 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
I20240502 02:01:11.190083 13094 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:11.247764 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:11.289786 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:11.296797 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:11.380096 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:11.383671 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:11.419812 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:11.460999 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:11.559985 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:11.567473 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:11.634178 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:11.643769 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:11.725926 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:11.729405 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
I20240502 02:01:11.732642 13628 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:11.747932 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:11.788370 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:11.833863 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:11.896193 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:11.902581 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:11.922755 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:11.986630 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:12.018832 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:12.023522 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:12.057781 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:12.064281 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:12.236585 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
I20240502 02:01:12.327178 13755 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:12.408263 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:12.490365 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:12.592924 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:12.603938 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:12.661662 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:12.737933 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:12.792397 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:12.809216 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:12.835309 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:12.842132 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
I20240502 02:01:12.944013 13755 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:12.989279 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:12.995657 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:13.026873 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:13.035184 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:13.043803 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:13.048686 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:13.052100 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:13.153513 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 02:01:13.260517 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:13.358893 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:13.447113 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:13.458422 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
I20240502 02:01:13.465186 13964 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:13.477615 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:13.516383 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:13.568260 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:13.572180 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:13.684036 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:13.756718 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:13.951448 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:13.956280 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:13.978075 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
I20240502 02:01:14.072470 13755 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:14.088982 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:14.110175 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:14.132866 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:14.135612 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:14.137053 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:14.163342 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:14.187076 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:14.197171 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:14.247010 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:14.252943 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:14.295799 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:14.428522 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:14.466702 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:14.510983 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:14.521765 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:14.539224 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:14.660200 14224 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:14.669826 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:14.739089 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:14.747148 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:14.748961 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:14.855638 14901 catalog_manager.cc:5014] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 3f938c5af51a43c9a23b753c4991e41d with cas_config_opid_index 15: ChangeConfig:ADD_PEER:NON_VOTER failed with leader 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323) due to error UNKNOWN_ERROR; will retry: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }.
Pending config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
W20240502 02:01:15.002700 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:15.027805 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:15.034534 14994 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 99 similar messages]
W20240502 02:01:15.045078 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:15.052631 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:15.086279 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:15.138945 14224 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:15.161042 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:15.183099 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:15.242298 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:15.311780 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:15.315711 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 77 similar messages]
W20240502 02:01:15.350214 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:15.409180 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:15.452307 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:15.462692 15266 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 89 similar messages]
W20240502 02:01:15.475455 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 70 similar messages]
W20240502 02:01:15.483459 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:15.500451 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:15.520949 14224 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:15.579062 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:15.662142 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 02:01:15.663452 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:15.685286 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 02:01:15.754102 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:15.764451 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:15.802855 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:15.863564 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
I20240502 02:01:15.909233 13755 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:15.957301 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:15.972028 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:16.032047 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:16.067693 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:16.086687 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:16.087405 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:16.215492 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
I20240502 02:01:16.464805 14591 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:16.476585 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:16.514283 14901 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 2 similar messages]
W20240502 02:01:16.515095 14901 catalog_manager.cc:4573] TS d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 21e2e55d1a0d4b6eb18b4ade9468607f: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:16.569157 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:16.571614 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:16.575438 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:16.589152 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:16.606670 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:16.652601 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:16.659103 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:16.733266 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:16.755206 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:16.759224 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:16.783896 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:16.829342 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:16.850805 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
I20240502 02:01:16.855769 14591 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:16.883270 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:16.958998 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:17.088060 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:17.126659 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:17.165644 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:17.179333 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:17.186337 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:17.256690 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:17.263764 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:17.274627 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
I20240502 02:01:17.407485 14800 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:17.418429 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:17.447350 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:17.582545 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:17.609314 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:17.612530 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:17.658429 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:17.683900 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:17.727767 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:17.746806 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:17.864482 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:17.952879 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
I20240502 02:01:18.000093 15117 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:18.041986 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:18.064390 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:18.122174 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 02:01:18.137017 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:18.175045 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:18.312469 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:18.343075 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:18.401655 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:18.411892 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:18.421875 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:18.466318 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:18.536984 15622 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:18.559540 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:18.599462 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:18.634076 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:18.701674 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:18.766657 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:18.993142 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:19.005590 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
I20240502 02:01:19.058240 15622 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:19.058689 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:19.069109 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:19.106797 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:19.115068 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:19.129714 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:19.148247 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:19.195283 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:19.205267 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:19.245638 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:19.290818 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:19.307988 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:19.375411 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:19.449066 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:19.458855 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:19.474376 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:19.526942 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:19.571203 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:19.619794 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:19.640570 15863 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:19.672739 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:19.782492 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:19.785394 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:19.799103 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:19.889043 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:19.910637 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:19.949249 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:19.969918 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:19.974316 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:20.025197 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:20.040962 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 101 similar messages]
W20240502 02:01:20.041893 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:20.042259 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
I20240502 02:01:20.074158 15622 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:20.075665 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:20.217025 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:20.327750 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 02:01:20.347229 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:20.351032 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:20.449540 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 02:01:20.481549 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 92 similar messages]
W20240502 02:01:20.501307 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:20.589556 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:20.652581 15127 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 70 similar messages]
I20240502 02:01:20.657359 16077 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:20.679654 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:20.714421 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:20.764106 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:20.786018 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 52 similar messages]
W20240502 02:01:20.790884 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:20.819178 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:20.820000 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:20.910055 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:21.003268 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:21.009668 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:21.125761 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:21.169785 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:21.223022 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:21.238253 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
I20240502 02:01:21.252635 16180 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:21.267326 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:21.350678 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:21.434090 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:21.520660 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:21.589691 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:21.619833 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:21.643344 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:21.687758 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:21.728644 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:21.754490 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
I20240502 02:01:21.775415 16284 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:21.818013 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:21.819628 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:21.835975 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:21.868063 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:21.875764 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:21.899776 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:21.901304 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:22.001741 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:22.105597 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:22.148546 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:22.177831 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:22.188771 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:22.235260 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
I20240502 02:01:22.281203 16392 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:22.322732 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:22.351099 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:22.386744 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:22.399871 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:22.403506 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:22.430974 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:22.481449 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:22.560745 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
I20240502 02:01:22.571846 15758 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:01:22.571998 15758 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:01:22.572500 15758 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:01:22.572697 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 02:01:22.572844 15465 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 02:01:22.573473 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:22.573612 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:22.573683 15263 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:01:22.573835 15758 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:01:22.626890 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
I20240502 02:01:22.676101 16392 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:22.789566 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:22.841104 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:22.852394 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 02:01:22.963677 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:22.972038 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:23.015738 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:23.059049 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
I20240502 02:01:23.130688 16392 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:23.270992 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:23.283330 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:23.309077 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:23.322484 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:23.342391 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:23.356462 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:23.402913 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:23.537324 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:23.565378 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:23.601467 16392 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:23.624334 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:23.687003 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:23.817499 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:23.838315 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:23.898659 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:23.905166 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:23.952059 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:23.967427 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:23.970584 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
I20240502 02:01:24.020903 16392 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:24.062986 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:24.259380 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:24.295732 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:24.307425 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:24.345803 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:24.364692 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:24.367753 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:24.418637 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:24.427425 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:24.484709 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:24.512666 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
I20240502 02:01:24.529743 16986 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:24.553867 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:24.676105 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:24.713147 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:24.724445 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:24.776032 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:24.791090 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:24.807430 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:24.843887 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:24.846283 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:24.927804 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:24.996692 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:25.048123 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 96 similar messages]
W20240502 02:01:25.048668 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:25.070827 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:25.075767 16986 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:25.093154 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:25.201702 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:25.230762 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:25.342989 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 02:01:25.404409 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 02:01:25.434690 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:25.453617 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:25.479559 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:25.498840 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:25.500487 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:25.528936 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 02:01:25.529629 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:25.662652 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:25.683616 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 70 similar messages]
I20240502 02:01:25.698144 17090 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:25.709532 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:25.800146 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:25.875123 16773 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 55 similar messages]
W20240502 02:01:25.900068 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:25.905104 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:26.032186 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:26.039645 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:26.051230 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:26.058111 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:26.157594 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:26.222064 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:26.231436 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:26.245148 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:26.253825 17090 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:26.296062 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:26.521445 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:26.547552 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:26.613552 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:26.625106 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:26.640384 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
I20240502 02:01:26.665808 17359 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:26.760649 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:26.768803 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:26.799189 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:26.808094 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:26.852012 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:26.889411 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:27.006458 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:27.019526 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:27.081996 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:27.107230 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:27.134614 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:27.175829 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:27.190778 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
I20240502 02:01:27.270048 17359 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:27.290968 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:27.306171 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:27.345948 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:27.350533 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:27.356920 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:27.410043 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:27.427960 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:27.492363 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:27.522061 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 02:01:27.567943 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:27.750876 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:27.777262 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:27.818357 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
I20240502 02:01:27.837831 17359 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:27.895426 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:27.925094 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:27.930285 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:28.022331 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:28.035733 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:28.036486 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:28.061048 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:28.210582 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:28.327306 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:28.381247 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:28.382409 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:28.400871 17359 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:28.420847 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:28.488209 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:28.520577 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:28.582860 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:28.621376 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:28.625715 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:28.685639 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:28.837087 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:28.866470 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:28.962510 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
I20240502 02:01:28.979115 17932 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:29.021793 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:29.055653 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:29.085424 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:29.158108 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:29.175788 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:29.199751 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:29.290796 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:29.293659 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:29.318368 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
I20240502 02:01:29.397521 17932 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:29.453783 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:29.507035 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:29.575565 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:29.586422 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:29.627103 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:29.665027 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:29.680366 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:29.732208 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:29.778945 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
I20240502 02:01:29.786777 17359 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:29.787971 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:29.803134 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:29.828630 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:29.883517 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:29.887974 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:29.951510 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:29.988281 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:30.038239 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 02:01:30.054253 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:30.092515 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 101 similar messages]
W20240502 02:01:30.093185 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:30.204058 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
I20240502 02:01:30.243605 17359 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:30.249527 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:30.368891 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:30.422755 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:30.431190 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:30.440315 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 02:01:30.520545 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:30.569613 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 02:01:30.590008 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:30.603484 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:30.707798 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
I20240502 02:01:30.718353 17932 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:30.747157 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:30.757381 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 71 similar messages]
W20240502 02:01:30.758016 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:30.793479 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:30.862546 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:30.916983 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 51 similar messages]
W20240502 02:01:30.954489 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:30.964061 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:30.987735 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:31.075599 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:31.153203 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:31.203718 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:31.272516 17932 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:31.400419 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:31.412966 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:31.484968 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:31.495594 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:31.525574 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:31.574753 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:31.620373 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:31.665004 17932 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:31.681727 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:31.716145 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
I20240502 02:01:31.744944 16495 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:01:31.745090 16495 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:01:31.745704 16495 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:01:31.745967 15328 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:01:31.746114 15328 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:01:31.746577 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:31.746950 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:31.747012 15399 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:01:31.747161 16495 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:01:31.762728 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:31.822404 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:31.839027 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:31.879467 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:31.949018 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:31.976365 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:32.057044 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:32.062724 17932 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:32.091814 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:32.096975 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:32.112687 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:32.133651 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:32.178076 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:32.288200 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:32.361961 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:32.389020 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
I20240502 02:01:32.456045 18399 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:32.487386 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 02:01:32.491202 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:32.534715 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:32.579563 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:32.597127 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:32.632773 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:32.669884 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:32.714998 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:32.802469 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:32.803308 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
I20240502 02:01:32.878760 17932 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:32.906191 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:33.010883 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:33.022305 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:33.124857 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:33.127351 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:33.139664 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:33.186100 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:33.233367 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:33.269524 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:33.312598 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:33.325855 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
I20240502 02:01:33.383677 18810 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:33.435046 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:33.479571 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:33.506557 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:33.564373 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:33.727845 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:33.785122 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:33.811511 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
I20240502 02:01:33.848085 17932 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:33.918421 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:33.944123 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:33.953681 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:33.988689 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:34.047253 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:34.075327 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:34.127689 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:34.181274 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:34.181439 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:34.232892 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
I20240502 02:01:34.241410 18810 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:34.319044 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:34.347869 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:34.381068 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:34.383252 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:34.444890 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:34.498558 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:34.499634 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:34.534402 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:34.737783 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:34.741540 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
I20240502 02:01:34.764858 17932 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:34.808501 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:34.810921 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:34.864931 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:34.868361 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:34.901234 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:34.942107 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:34.949748 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 02:01:35.020961 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:35.111773 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 100 similar messages]
W20240502 02:01:35.138491 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:35.178872 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:35.274884 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:35.278923 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:35.308404 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:35.316421 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:35.355332 17932 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:35.393106 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:35.455852 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 02:01:35.638456 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:35.649109 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 90 similar messages]
W20240502 02:01:35.649719 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:35.701692 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:35.740145 17932 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:35.745301 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:35.755246 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:35.767251 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:35.773717 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:35.857277 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 68 similar messages]
W20240502 02:01:35.857829 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:35.862643 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:35.883037 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:35.913353 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:36.033421 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:36.034911 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 02:01:36.049700 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:36.100205 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:36.173025 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:36.195266 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
I20240502 02:01:36.260841 19414 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:36.306115 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:36.379145 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:36.384222 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:36.472036 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:36.528139 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:36.534330 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:36.653898 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:36.686455 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:36.707702 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:36.792512 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
I20240502 02:01:36.838891 19414 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:36.864843 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:36.909082 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:36.954562 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:36.976681 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:37.051314 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:37.135877 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:37.145260 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:37.163480 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:37.173334 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:37.204742 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:37.205312 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:37.217552 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
I20240502 02:01:37.256458 19414 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:37.269634 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 02:01:37.273442 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:37.383484 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:37.513967 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:37.520056 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:37.545926 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:37.653615 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:37.730172 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:37.768280 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:37.775276 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
I20240502 02:01:37.841130 19754 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:37.847128 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:37.874629 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:38.010949 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:38.051715 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:38.072393 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:38.108817 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:38.152155 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:38.276465 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
I20240502 02:01:38.306562 19754 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:38.308372 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:38.321094 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:38.432399 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:38.484747 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:38.510810 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:38.526880 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:38.570902 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:38.580070 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
I20240502 02:01:38.622152 19698 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:01:38.622273 19698 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:01:38.622710 19698 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:01:38.623023 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:01:38.623186 15329 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:01:38.623656 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:38.623999 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:38.624054 15399 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:01:38.624161 19698 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:01:38.658059 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:38.678709 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:38.709141 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:38.718712 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
I20240502 02:01:38.814500 19754 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:38.825820 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:38.887248 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:38.965803 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:39.047338 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:39.135275 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:39.138760 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:39.214805 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:39.253768 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
I20240502 02:01:39.365146 19754 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:39.380728 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:39.497457 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:39.501804 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:39.512754 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:39.531085 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:39.556859 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:39.610641 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:39.613581 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:39.717578 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:39.736788 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 02:01:39.740842 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
I20240502 02:01:39.786214 19754 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:39.828886 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:39.841774 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:39.877981 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:39.992820 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:39.996394 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:40.014573 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:40.029104 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:40.071568 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:40.144834 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 98 similar messages]
W20240502 02:01:40.145476 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:40.157465 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:40.257143 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:40.279336 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:40.365526 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
I20240502 02:01:40.389503 19754 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:40.396095 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:40.414716 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:40.552106 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 83 similar messages]
W20240502 02:01:40.552726 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:40.606060 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:40.608846 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:40.663650 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 89 similar messages]
W20240502 02:01:40.664269 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:40.762236 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:40.770644 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:40.889935 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 70 similar messages]
W20240502 02:01:40.890494 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:40.965108 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
I20240502 02:01:40.976110 20274 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:41.015703 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:41.018247 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:41.023917 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:41.080361 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:41.106504 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 51 similar messages]
W20240502 02:01:41.153203 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:41.233942 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:41.306725 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:41.337558 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:41.414126 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:41.433791 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:41.444523 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:41.454902 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:41.594197 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
I20240502 02:01:41.597931 20534 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:41.637617 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:41.667539 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:41.747017 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:41.805029 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:41.810974 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:41.859045 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:41.908252 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:41.928663 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:41.979619 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:42.080435 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:42.109511 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:42.122859 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
I20240502 02:01:42.126902 20534 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:42.135848 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:42.246867 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:42.254348 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:42.267114 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:42.378816 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 02:01:42.452322 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:42.465399 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:42.566025 20534 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:42.597833 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:42.630678 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:42.652889 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:42.659200 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:42.707991 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:42.745275 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:42.767812 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:42.799786 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:42.886360 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:42.904614 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:42.983034 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:43.003484 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:43.093966 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:43.108557 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:43.183578 20534 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:43.197594 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:43.245183 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:43.363446 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:43.391682 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:43.488894 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:43.576205 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:43.592729 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:43.683735 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:43.686239 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:43.707662 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:43.711664 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:43.737159 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:43.789736 20534 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:43.800400 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:43.803038 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:43.877164 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:43.905442 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:43.917929 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:43.926577 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:44.087882 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:44.214186 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:44.306954 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:44.340209 21104 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:44.341089 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:44.481415 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:44.503774 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:44.507386 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:44.533223 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:44.553220 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:44.597663 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:44.599300 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:44.623962 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:44.686723 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:44.690023 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 02:01:44.695766 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:44.766958 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:44.887416 21104 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:44.907784 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:45.017560 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:45.073452 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:45.091593 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:45.102825 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:45.159631 14994 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 96 similar messages]
W20240502 02:01:45.187814 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:45.262147 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:45.281092 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:45.321893 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:45.338369 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:45.343394 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
I20240502 02:01:45.395107 21311 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:45.406622 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:45.408258 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:45.474817 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:45.579872 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:45.654522 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 02:01:45.655121 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:45.797685 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 02:01:45.856072 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:45.938016 21104 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:45.939467 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 67 similar messages]
W20240502 02:01:45.940003 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:45.973317 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:45.991204 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:46.030450 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:46.051908 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:46.137949 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:46.172662 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 52 similar messages]
W20240502 02:01:46.173296 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:46.179268 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:46.190256 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:46.211380 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:46.250083 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
I20240502 02:01:46.319223 21414 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:46.422518 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:46.428169 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:46.446645 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:46.451642 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:46.506528 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:46.581549 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:46.717267 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:46.764357 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:46.855904 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:46.867870 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:46.881294 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
I20240502 02:01:46.893864 21622 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:46.912117 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:46.919620 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:46.955137 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:47.061173 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:47.115388 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:47.126734 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:47.142513 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:47.216766 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 02:01:47.341883 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:47.348259 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:47.364409 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:47.370510 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:47.501461 21622 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:47.589401 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:47.610489 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:47.634548 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:47.671976 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:47.691028 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:47.774427 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:47.803160 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:47.845207 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:47.849269 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:47.896521 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:47.934705 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:47.935855 21726 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:47.988241 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:47.990756 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:48.112954 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:48.178519 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:48.261490 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:48.306334 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:48.312825 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:48.340912 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:48.393146 21726 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:48.449888 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:48.478413 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:48.495601 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:48.511489 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:48.523509 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:48.580550 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:48.670321 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:48.717478 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:48.737272 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:48.803902 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:48.821197 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:48.918334 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:48.924163 21726 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:48.942656 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:49.095470 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:49.123505 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:49.133484 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:49.176810 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
I20240502 02:01:49.352789 21622 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:49.362226 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:49.376209 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:49.377949 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:49.391050 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:49.426355 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:49.427974 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:49.477666 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:49.552225 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:49.575495 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:49.658623 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:49.683327 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:49.714071 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:49.896006 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:49.902380 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 02:01:49.914602 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:49.916297 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
I20240502 02:01:49.962270 21622 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:49.982676 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:50.077327 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:50.097430 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:50.107725 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:50.124363 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:50.233378 14992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 102 similar messages]
W20240502 02:01:50.260247 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
I20240502 02:01:50.371160 21622 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:50.388669 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:50.393698 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:50.407312 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:50.462941 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:50.469882 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:50.490453 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:50.531416 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:50.630265 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:50.635301 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:50.718546 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 78 similar messages]
W20240502 02:01:50.719125 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:50.843267 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 90 similar messages]
I20240502 02:01:50.873839 22450 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:50.902490 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:50.928828 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:50.954547 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:51.004094 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 70 similar messages]
W20240502 02:01:51.004832 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:51.045007 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:51.123703 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:51.132855 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:51.143985 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:51.210624 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:51.305500 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 49 similar messages]
W20240502 02:01:51.306141 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:51.339967 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:51.348014 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:51.353654 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:51.374409 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:51.451656 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:51.497857 22450 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:51.545647 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:51.629338 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:51.644098 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:51.811429 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:51.873832 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:51.909093 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
I20240502 02:01:51.947153 21622 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:51.955097 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:51.967625 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:51.972095 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:52.033557 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:52.045311 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:52.089888 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:52.113895 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:52.161661 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:52.207976 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:52.282907 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:52.466228 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:52.498549 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
I20240502 02:01:52.524273 22450 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:52.536080 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 02:01:52.667449 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:52.685267 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:52.732095 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:52.755072 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:52.766002 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:52.804212 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:52.836035 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:52.877339 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
I20240502 02:01:52.959246 21622 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:52.978799 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:53.017341 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:53.035156 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:53.041849 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:53.195551 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:53.301059 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:53.306049 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:53.336742 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:53.352886 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:53.396109 22450 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:53.442582 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:53.452204 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:53.541961 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:53.573951 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:53.669142 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:53.681733 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:53.688740 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:53.723618 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:53.745270 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:53.745337 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:53.756378 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:53.771656 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:53.892812 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:53.910248 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:53.942384 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
I20240502 02:01:53.988643 22450 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:54.090467 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:54.166101 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:54.168092 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:54.168149 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:54.245564 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:54.331575 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:54.351949 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:54.395804 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
I20240502 02:01:54.399346 22450 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:54.512326 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:54.574544 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:54.587750 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:54.607534 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:54.629478 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:54.745292 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:54.783582 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:54.804267 22450 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:54.807054 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:54.822131 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 02:01:54.878906 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:54.982478 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:55.011622 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:55.166891 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:55.170295 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:55.202765 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:55.259779 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
I20240502 02:01:55.262449 22450 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:55.281718 14994 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 104 similar messages]
W20240502 02:01:55.285621 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:55.303406 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:55.420228 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:55.460422 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:55.481079 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:55.583309 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
I20240502 02:01:55.612964 23127 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:01:55.613094 23127 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:01:55.613622 23127 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:01:55.614208 15465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 02:01:55.614351 15465 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 02:01:55.615123 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:55.615440 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:55.615512 15263 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:01:55.615828 23127 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:01:55.621418 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:55.669342 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:55.678252 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:55.688953 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:55.691643 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
I20240502 02:01:55.725776 23181 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:55.739691 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:55.751303 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 78 similar messages]
W20240502 02:01:55.845666 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:55.855463 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 02:01:55.898386 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:55.929551 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:55.954201 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:55.985226 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:56.048676 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 70 similar messages]
W20240502 02:01:56.066462 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:56.169108 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:56.190681 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:56.235112 23549 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:56.235558 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:56.237339 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:56.306977 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:56.363093 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:56.383200 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 02:01:56.383842 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:56.409845 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:56.475917 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:56.592927 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:56.647051 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:56.657855 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:56.664526 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:01:56.708838 23549 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:56.744298 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:56.761729 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:56.841030 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:56.843709 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:56.856982 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:56.983150 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:56.994382 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:01:57.179769 23181 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:57.190388 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:57.247371 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:57.252719 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:57.309388 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:57.355340 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:57.400651 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:57.511891 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 02:01:57.546301 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:57.593024 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:57.598237 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:57.633831 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
I20240502 02:01:57.770224 23861 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:57.829005 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:57.832854 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:57.879290 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:57.925165 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:57.975425 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:57.992830 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:58.052805 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:58.162091 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:58.267421 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:58.283370 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:58.312577 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
I20240502 02:01:58.321924 23861 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:58.334766 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:58.377247 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:58.393654 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:58.396126 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:58.437151 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:58.438524 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:58.456123 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:58.486063 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:58.550310 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:58.550889 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:58.634554 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:58.732935 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:58.733021 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:58.745581 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
I20240502 02:01:58.778977 23964 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:58.790360 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:58.945897 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:58.965338 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:59.012674 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:59.062566 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:59.075292 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:01:59.117961 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:59.203722 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:59.251996 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:59.273365 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:59.310909 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:59.339571 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:01:59.350186 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:01:59.381150 23964 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:59.393347 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:59.508000 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:01:59.552327 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:01:59.785355 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:01:59.846657 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
I20240502 02:01:59.892079 23964 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:01:59.919862 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:01:59.956017 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:00.054522 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:00.102201 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:00.138175 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:02:00.163630 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:00.211844 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 02:02:00.289146 14994 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 99 similar messages]
W20240502 02:02:00.336676 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
I20240502 02:02:00.373617 23964 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:00.376538 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:00.388904 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:00.432888 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:00.476009 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:00.485785 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:00.529417 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:00.583141 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:00.595917 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:00.631536 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:00.766850 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:00.787535 15398 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 02:02:00.788134 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:00.831148 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
I20240502 02:02:00.841779 24315 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:00.893491 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 02:02:00.904175 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:00.922948 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:00.927091 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:01.003408 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:01.028187 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:01.095688 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:01.120676 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 02:02:01.121261 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:01.148077 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:01.194168 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:01.197214 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:01.207372 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:01.252902 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:01.312238 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
I20240502 02:02:01.314041 23964 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:01.342094 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:01.401656 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:01.424131 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:01.467031 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:01.506570 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:01.542356 16773 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 02:02:01.569793 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:01.573725 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:01.584852 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:01.607836 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:01.627550 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:01.637205 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
I20240502 02:02:01.716502 23964 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:01.721783 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
I20240502 02:02:02.005496 24018 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:02:02.005626 24018 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:02:02.006098 24018 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:02:02.006433 15464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 02:02:02.006557 15464 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 02:02:02.007251 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:02.007385 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:02.007438 15263 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:02:02.007560 24018 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:02:02.072993 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:02.087881 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:02.094305 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:02.107293 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:02:02.121100 24315 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:02.260185 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:02.388324 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:02.480377 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
I20240502 02:02:02.501022 24315 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:02.528488 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:02.547750 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:02.615804 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:02.637441 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:02.694818 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:02.740397 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 02:02:02.866009 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:02.931181 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:02.933190 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:02.946105 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:02:03.041393 23964 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:03.104110 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:03.129201 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:03.225790 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:03.245919 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:03.270192 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:03.285159 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:03.291579 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:03.345135 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:03.370718 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:03.457917 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
I20240502 02:02:03.513005 24315 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:03.513840 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:03.514375 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:03.542445 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:03.545435 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:03.580992 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:03.582886 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:03.592746 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:03.607475 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:03.678807 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:03.821744 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:03.829236 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:03.861725 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:03.862068 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:03.963605 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:03.970734 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:03.977031 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:04.032461 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:04.052088 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:04.076381 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
I20240502 02:02:04.084702 24315 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:04.096283 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:04.126495 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:04.166487 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:04.236775 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:04.300467 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:04.395081 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:04.510277 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:04.657883 24315 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:04.706156 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:04.730283 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:04.779049 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:04.849102 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:04.894827 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:04.971045 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:05.023767 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
I20240502 02:02:05.048899 25274 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:05.178000 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:05.185485 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:05.194867 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:05.235579 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 02:02:05.295572 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:05.317378 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 99 similar messages]
W20240502 02:02:05.345750 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:05.373962 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:05.425547 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:05.459609 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:05.515837 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:05.556478 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
I20240502 02:02:05.610893 25476 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:05.643617 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:05.737287 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:05.782119 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:05.812114 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:05.813814 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 02:02:05.828181 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:05.842733 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:05.870934 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:05.891259 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:05.938349 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 89 similar messages]
W20240502 02:02:05.953989 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:05.963433 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:05.989693 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:06.011940 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:06.057128 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:06.065490 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:06.074136 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:06.106333 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:06.175060 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
I20240502 02:02:06.228029 25632 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:06.236456 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:06.349828 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 71 similar messages]
W20240502 02:02:06.371300 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:06.375131 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:06.383380 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:06.433892 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:06.471876 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:06.553671 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:06.577569 24018 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:02:06.577701 24018 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:02:06.578146 24018 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:02:06.578438 15464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 02:02:06.578621 15464 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 02:02:06.579126 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:06.579418 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:06.579469 15263 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:02:06.579597 24018 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:02:06.580687 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:06.596683 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 48 similar messages]
W20240502 02:02:06.597272 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
I20240502 02:02:06.632993 25476 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:06.643389 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:06.703925 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:06.718518 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:06.772631 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:06.806561 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:06.902143 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:07.011252 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
I20240502 02:02:07.068701 25632 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:07.080415 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:07.255499 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:07.330777 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:07.374051 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:07.464326 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
I20240502 02:02:07.465191 25632 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:07.615757 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:07.620857 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:07.655143 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:07.672808 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:07.674887 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:07.692745 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 02:02:07.711895 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:07.814129 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:07.862152 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
I20240502 02:02:07.966312 25952 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:07.998886 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:08.054283 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:08.106312 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:08.113842 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:08.125360 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:08.148687 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:08.187439 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:08.272665 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:08.341989 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:08.347528 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:08.408861 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
I20240502 02:02:08.410852 25952 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:08.437500 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:08.500924 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:08.513831 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:08.518795 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:08.524686 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:08.551330 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:08.554138 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:08.629175 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:08.679723 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:08.740307 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:08.779678 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:08.793773 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:08.807790 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:08.809309 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:08.868938 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:08.902127 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:08.911136 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:08.936553 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
I20240502 02:02:08.941632 26154 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:08.983639 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:09.074940 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:09.143357 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:09.401216 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:09.410164 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:09.417795 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:09.468755 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
I20240502 02:02:09.477890 25952 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:09.481808 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:09.576256 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:09.597718 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:09.821849 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:09.827781 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:09.861503 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:09.899752 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:09.946130 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:02:09.989226 25952 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:10.120705 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:10.150692 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:10.256876 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:10.295351 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:10.336499 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:10.340234 14992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 100 similar messages]
I20240502 02:02:10.377200 25952 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:10.405958 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:10.421977 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:10.470007 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:10.557972 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:10.618324 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 02:02:10.669607 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:10.735486 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:10.745695 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:10.795724 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
I20240502 02:02:10.824759 25952 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:10.831813 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:10.840209 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:10.914191 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 02:02:10.935067 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:10.984256 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:11.044684 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 94 similar messages]
W20240502 02:02:11.048137 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:11.054737 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:11.104746 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:11.113950 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:11.122136 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:11.137640 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:11.156752 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
I20240502 02:02:11.208889 25952 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:11.221087 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:11.228999 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:11.241012 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:11.273969 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:11.280575 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:11.299816 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:11.321803 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:11.329808 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:11.389437 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:11.413734 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:11.423971 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:11.527123 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:11.559275 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:11.581568 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:11.586012 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 70 similar messages]
W20240502 02:02:11.686759 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 48 similar messages]
W20240502 02:02:11.710618 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:11.738762 26367 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:11.788611 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:11.820919 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:11.941632 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:11.941718 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:12.004480 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:12.005637 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:12.114470 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:12.182677 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:12.283815 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
I20240502 02:02:12.349061 26367 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:12.430923 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:12.452876 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:12.556749 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:12.582934 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:12.626894 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:12.784521 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:12.803025 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:12.890055 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
I20240502 02:02:12.941473 26367 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:12.955130 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:12.969105 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:13.052140 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:13.065469 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:13.084172 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:13.130839 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:13.195250 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:13.357285 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:02:13.382872 26367 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:13.396245 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:13.409190 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 02:02:13.424203 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:13.447409 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:13.473980 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:13.495751 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:13.503432 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:13.644299 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:13.666649 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:13.693002 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:13.703177 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:13.732970 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:13.733397 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:13.747974 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:13.765022 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:13.779747 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:13.805168 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:13.878324 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:13.905897 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:13.929334 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:13.944049 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:13.951004 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:02:13.964990 26367 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:13.987072 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:14.001165 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:14.009586 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:14.239549 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:14.255331 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:14.292167 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:02:14.339980 26367 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:14.434237 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:14.573557 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:14.623872 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:14.658670 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:14.731739 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:14.820334 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:14.825179 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:14.855302 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:14.887349 14901 catalog_manager.cc:5014] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 3f938c5af51a43c9a23b753c4991e41d with cas_config_opid_index 15: ChangeConfig:ADD_PEER:NON_VOTER failed with leader 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323) due to error UNKNOWN_ERROR; will retry: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }.
Pending config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 02:02:14.904570 27413 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:14.929852 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:15.136349 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:15.150427 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:15.178637 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:15.187101 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:15.198124 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:15.224092 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
I20240502 02:02:15.308830 26367 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:15.403388 14994 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 101 similar messages]
W20240502 02:02:15.404042 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:15.435508 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:15.544422 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:15.582346 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 02:02:15.663630 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:15.672206 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:15.731932 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:15.772258 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
I20240502 02:02:15.775357 27413 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:15.776494 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:15.785682 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:15.852284 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:15.868597 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:15.937491 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:16.002130 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 02:02:16.018806 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:16.019604 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:16.063997 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:16.102903 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:16.103111 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 89 similar messages]
W20240502 02:02:16.127142 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:16.159801 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:16.205188 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:16.216141 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:16.237574 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:16.334815 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
I20240502 02:02:16.363476 27413 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:16.384277 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:16.384974 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:16.391777 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:16.447198 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:16.466226 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:16.522204 14901 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 2 similar messages]
W20240502 02:02:16.522842 14901 catalog_manager.cc:4573] TS d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 21e2e55d1a0d4b6eb18b4ade9468607f: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:16.532208 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:16.548717 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:16.554056 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:16.569804 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:16.650833 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:16.708204 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 72 similar messages]
W20240502 02:02:16.720467 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:16.832330 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 02:02:16.845118 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:16.860507 27709 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:16.869784 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:16.990487 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:16.999589 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:17.091809 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:17.112476 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:17.261943 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:17.295046 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:17.319638 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:17.374806 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
I20240502 02:02:17.438521 27413 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:17.494565 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:17.514183 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:17.528256 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:17.552767 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:17.632727 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:17.852267 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:17.869031 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:17.895836 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 02:02:17.902966 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:17.935086 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
I20240502 02:02:17.976825 28037 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:18.049278 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:18.084745 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:18.104341 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:18.132243 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:18.174095 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:18.233660 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:18.242355 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:18.252820 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:18.310506 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:18.371538 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
I20240502 02:02:18.396049 28037 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:18.422947 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:18.469168 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:18.567688 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:18.584461 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:18.644403 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:18.654263 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:18.671378 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:18.746829 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:18.792546 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:18.796108 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
I20240502 02:02:18.840930 27413 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:18.869128 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:18.897675 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:18.927170 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:18.928567 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:18.973099 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:19.013507 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:19.107638 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:19.149389 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:19.169603 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:19.177892 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:19.228654 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:19.402169 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:19.419310 28037 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:19.449200 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:19.470940 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:19.534655 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:19.623190 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:19.657945 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:19.717239 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:19.736548 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:19.858150 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:19.903718 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:19.984077 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:19.989331 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:20.017283 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
I20240502 02:02:20.043670 28453 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:20.151669 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:20.178622 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:20.212369 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:20.320838 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:20.412371 14994 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 99 similar messages]
W20240502 02:02:20.428292 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:20.467085 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 02:02:20.513831 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:20.555110 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:20.566100 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
I20240502 02:02:20.568445 28558 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:20.625474 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:20.693785 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:20.746057 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:20.775923 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:20.775964 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:20.814498 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:20.869550 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:20.957630 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:21.019721 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 02:02:21.020325 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:21.065918 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:21.091075 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:21.108777 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 90 similar messages]
W20240502 02:02:21.126850 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
I20240502 02:02:21.186376 28708 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:21.194376 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:21.257196 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:21.267144 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:21.396577 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:21.415894 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:21.430866 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:21.452207 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:21.457171 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:21.502472 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:21.517405 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:21.554571 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:21.591233 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:21.591619 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:21.617170 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
I20240502 02:02:21.618749 28708 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:21.729159 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:21.732343 15127 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 02:02:21.822676 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:21.838567 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:21.868623 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:21.882386 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 02:02:21.965220 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:21.979439 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:21.985167 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:22.007108 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
I20240502 02:02:22.061198 28819 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:22.110996 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:22.277272 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:22.325551 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:22.436667 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:22.464046 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:22.504640 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:02:22.552323 28819 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:22.586141 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:22.593549 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:22.602087 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:22.647519 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:22.677541 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:22.899997 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:22.913046 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:22.919165 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
I20240502 02:02:22.956949 28558 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:22.970528 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 02:02:23.093915 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:23.116123 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:23.124490 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:23.144704 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:23.147220 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:23.163956 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:23.192862 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:23.241389 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:23.325781 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:23.346505 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:23.506852 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:02:23.511504 28558 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:23.554272 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:23.576962 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:23.617024 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:23.645249 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:23.710774 28611 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:02:23.710916 28611 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:02:23.711513 28611 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:02:23.712117 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:02:23.712262 15329 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:02:23.712970 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:23.714144 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:02:23.714218 15399 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:02:23.714345 28611 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:02:23.736555 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:23.791306 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:23.809615 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:23.840867 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:23.852160 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:23.869647 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:23.870005 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
I20240502 02:02:23.888203 28558 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:23.904625 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:24.048399 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:24.059110 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:24.067975 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:24.102524 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:24.168715 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:24.298729 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:24.303479 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
I20240502 02:02:24.321964 28558 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:24.455825 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:24.474992 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:24.480831 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:24.525352 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:24.561271 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:24.602437 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:24.638335 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:24.672564 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:24.848138 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
I20240502 02:02:24.930845 29431 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:24.989912 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:24.994257 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:25.023167 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:25.035024 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:25.108506 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:25.175145 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:25.227460 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:25.260303 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:25.269670 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:25.396693 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 02:02:25.414950 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 104 similar messages]
W20240502 02:02:25.415570 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:02:25.446233 29431 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:25.483386 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:25.526971 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:25.531701 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:25.539203 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:25.572129 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:25.597976 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:25.748327 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:25.771855 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:25.776818 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:25.860616 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:25.908425 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:25.916253 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:25.930464 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:25.972960 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:25.989320 29431 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:26.054049 15398 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 84 similar messages]
W20240502 02:02:26.104585 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:26.115880 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 84 similar messages]
W20240502 02:02:26.187060 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:26.266520 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:26.268316 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:26.270494 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:26.330052 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:26.350047 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:26.418028 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:26.424389 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:26.435168 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:26.448422 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:26.519718 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:26.565124 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
I20240502 02:02:26.571890 29431 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:26.599650 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:26.635008 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:26.731559 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:26.817168 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 02:02:26.824517 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:26.830193 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:26.894091 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:26.943442 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:27.010828 16773 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 52 similar messages]
W20240502 02:02:27.012249 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:27.018913 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
I20240502 02:02:27.193337 29895 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:27.251309 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:27.270766 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:27.272969 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:27.286007 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:27.332724 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:27.348222 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:27.505425 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:27.653036 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:27.698403 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:27.720631 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:27.744562 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:27.799141 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:27.807548 29648 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:27.835978 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:27.853844 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:27.875607 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:27.936671 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:27.944108 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 02:02:27.964108 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:27.967911 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:28.052999 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:28.081838 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:28.155295 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:28.183033 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:28.191519 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
I20240502 02:02:28.206317 29895 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:28.311829 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:28.334678 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:28.348950 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:28.387144 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:28.440251 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:28.458118 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:28.570488 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:28.697530 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:28.727150 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:28.758957 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
I20240502 02:02:28.812122 29895 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:28.855453 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:28.876024 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:28.897610 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:28.972798 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:28.994611 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:29.003599 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:29.049810 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:29.078564 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:29.113524 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:29.251071 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
I20240502 02:02:29.282343 30242 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:29.325295 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:29.392803 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:29.438568 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:29.477707 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:29.592891 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:29.615420 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:29.689488 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:29.747124 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:29.796844 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
I20240502 02:02:29.832342 30242 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:29.866636 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:29.872138 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:29.906026 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:29.977769 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:29.982409 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:30.056586 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:30.132460 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:30.168804 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:30.292318 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
I20240502 02:02:30.322023 30451 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:30.366158 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:30.376958 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:30.398644 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:30.456539 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 101 similar messages]
W20240502 02:02:30.457564 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:30.469121 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:30.490989 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 02:02:30.557701 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:30.580986 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:30.642164 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:30.653887 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:30.662039 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:30.690342 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:30.764959 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
I20240502 02:02:30.776782 30451 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:30.819481 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:30.859551 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:30.873916 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:30.987118 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:30.987113 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:31.003862 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:31.033969 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:31.128717 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 89 similar messages]
I20240502 02:02:31.183269 30451 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:31.216920 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:31.239223 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:31.288062 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:31.292279 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 02:02:31.303791 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:31.432813 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:31.487455 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:31.502810 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:31.503726 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:31.525229 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:31.550803 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:31.562011 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:31.568503 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:31.628036 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
I20240502 02:02:31.688894 30822 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:31.850195 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 70 similar messages]
W20240502 02:02:31.851008 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:31.879777 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:31.912868 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:32.017794 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:32.057555 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 49 similar messages]
W20240502 02:02:32.076702 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:32.268275 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
I20240502 02:02:32.277887 30926 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:32.320940 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:32.326936 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:32.332690 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:32.417743 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:32.435750 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:32.435871 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:32.454957 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:32.555150 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:32.565420 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:32.578171 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:32.590342 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:02:32.766130 30926 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:32.797466 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:32.826613 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 02:02:32.890182 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:32.895247 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:32.952373 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:32.954382 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:32.962352 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:32.986490 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:33.004737 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:33.149348 30822 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:33.164253 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:33.244822 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:33.261809 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:33.264542 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:33.274065 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:33.277985 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:33.295285 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:33.330516 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:33.420665 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:33.439008 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:33.573834 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:33.578174 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:33.641448 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:33.680794 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:33.683283 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
I20240502 02:02:33.722223 30822 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:33.739981 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:33.797794 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:33.939409 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:34.040652 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:34.053934 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:34.071947 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:34.091876 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:34.094746 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:34.108386 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:34.207187 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:34.274127 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:34.274933 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
I20240502 02:02:34.283375 31344 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:34.341128 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:34.479749 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:34.666420 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:34.677681 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:34.678400 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:34.727609 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
I20240502 02:02:34.737200 31344 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:34.760798 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:34.921305 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:34.925787 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:34.977039 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:35.008939 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:35.076682 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:35.081904 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:35.130195 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:35.142845 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
I20240502 02:02:35.174515 30822 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:35.204181 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 02:02:35.278875 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:35.304646 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:35.315982 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:35.393966 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:35.416615 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:35.442853 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:35.452610 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:35.563657 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 100 similar messages]
W20240502 02:02:35.564594 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
I20240502 02:02:35.611301 31541 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:35.659106 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:35.676025 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:35.716161 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:35.753965 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:35.755134 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:35.772912 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:35.801128 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:35.843971 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:35.948585 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:36.062506 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:36.096328 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:36.112468 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:36.142398 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 89 similar messages]
I20240502 02:02:36.183907 31344 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:36.269855 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:36.283298 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:36.286924 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:36.291287 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:36.306556 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 02:02:36.308586 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:36.360915 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:36.461735 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:36.468139 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:36.561236 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:36.594277 31541 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:36.685235 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:36.710669 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:36.734315 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:36.746771 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:36.767335 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:36.815685 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:36.818135 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
I20240502 02:02:37.000383 31541 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:37.005059 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 02:02:37.005640 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:37.084142 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 02:02:37.139016 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:37.161072 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:37.281149 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:37.295801 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:37.355681 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:37.406337 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:37.488548 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
I20240502 02:02:37.511503 31541 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:37.537076 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:37.558357 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:37.587077 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:37.597266 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:37.612620 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:37.666378 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:37.694255 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:37.739197 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:37.832619 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:37.837388 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 02:02:37.960009 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:37.995729 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:38.006928 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:38.017999 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:38.037508 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:38.047070 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
I20240502 02:02:38.115135 31541 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:38.120344 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:38.140831 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:38.166632 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:38.169677 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:38.324985 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:38.394364 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:38.514267 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:38.519034 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:38.528296 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:38.535233 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:38.567466 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:38.625533 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
I20240502 02:02:38.628422 32225 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:38.631518 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:38.655856 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:38.686648 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:38.710331 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:38.978426 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:39.007901 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:39.026369 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
I20240502 02:02:39.093995 32225 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:39.119817 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:39.150537 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:39.175444 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:39.231001 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:39.249462 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:02:39.279059 32276 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:02:39.279201 32276 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:02:39.279758 32276 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:02:39.280092 15464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 02:02:39.280243 15464 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 02:02:39.280719 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:39.280812 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:02:39.280892 15266 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:02:39.281044 32276 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:02:39.295567 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:39.307468 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:39.438558 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:39.500643 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:39.567627 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:39.587175 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
I20240502 02:02:39.640388 32225 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:39.662611 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:39.736292 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:39.821149 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:39.853296 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:40.035693 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:40.046820 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:40.084417 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:40.108058 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
I20240502 02:02:40.108999 32435 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:40.116251 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:40.133654 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:40.170073 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:40.177120 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 02:02:40.196063 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:40.215720 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:40.284826 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:40.293834 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:40.441438 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:40.470225 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:40.471024 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
I20240502 02:02:40.529546 32435 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:40.568327 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:40.583057 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:40.590286 14992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 103 similar messages]
W20240502 02:02:40.696707 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:40.700131 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:40.720551 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:40.769716 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:40.777493 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:40.914815 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:40.930323 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
I20240502 02:02:41.097152 32435 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:41.114415 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:41.133263 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:41.141433 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:41.150233 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 89 similar messages]
W20240502 02:02:41.183990 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:41.211545 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:41.226122 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:41.263540 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:41.325070 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:41.330323 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 02:02:41.350338 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:41.404786 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:41.433763 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:41.487684 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:41.492995 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:41.534495 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
I20240502 02:02:41.562681 32435 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:41.617381 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:41.617578 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:41.782627 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:41.832432 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:41.903635 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:41.924018 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
I20240502 02:02:41.981745 32435 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:42.103224 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:42.110875 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 71 similar messages]
W20240502 02:02:42.111541 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:42.138163 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:42.189865 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 51 similar messages]
W20240502 02:02:42.190253 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:42.337182 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:42.343513 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:42.431516 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:42.461944 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
I20240502 02:02:42.507654 32435 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:42.529299 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:42.536087 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:42.614336 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:42.640894 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:42.657737 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 02:02:42.704922 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:42.756075 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:42.793279 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:42.859141 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:42.860397 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:42.930383 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:43.017006 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:43.039559 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
I20240502 02:02:43.044530 32435 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:43.058437 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:43.111748 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:43.128320 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:43.250447 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:43.295078 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:43.316579 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:43.316843 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:43.397905 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:43.402984 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:43.468931 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:43.482241 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:43.504032 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:43.541042 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:02:43.639480 797 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:43.695793 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:43.772900 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:43.793496 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:43.798985 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:43.800056 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:43.851470 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:43.919870 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:43.931583 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:43.974805 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:43.981863 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
I20240502 02:02:44.016680 32435 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:44.122392 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:44.208320 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:44.226519 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:44.274405 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:44.288882 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:44.308854 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:44.359261 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:44.552937 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:44.609326 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
I20240502 02:02:44.632894 32435 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:44.703460 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:44.768450 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:44.796078 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:44.878836 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:44.886781 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:44.923395 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:45.067569 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:45.098524 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:45.098788 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:45.162367 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:45.178370 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:45.192517 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:45.214855 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:45.230370 1108 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:45.285189 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:45.295609 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:45.358376 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:45.381479 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:45.386885 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:45.413936 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:45.439882 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 02:02:45.510736 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:45.568869 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:45.694011 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 101 similar messages]
W20240502 02:02:45.718454 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:02:45.744092 1211 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:45.752528 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:45.756990 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:45.771448 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:45.900781 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:45.982257 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:46.007748 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:46.094138 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:46.099829 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:46.142299 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:46.170228 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
I20240502 02:02:46.217278 1108 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:46.220710 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:46.245216 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:46.246048 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:46.250385 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 89 similar messages]
W20240502 02:02:46.251025 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:46.274947 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:46.301216 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:46.460642 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 82 similar messages]
W20240502 02:02:46.465729 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:46.488399 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:46.519183 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:46.532861 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:46.539513 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:46.608101 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
I20240502 02:02:46.617677 1211 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:46.709045 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:46.738682 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:46.831596 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:46.980815 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:47.071671 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:47.159886 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:47.187000 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 68 similar messages]
I20240502 02:02:47.209834 1521 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:47.252070 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 02:02:47.290146 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:47.323719 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:47.381258 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:47.447315 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:47.457142 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:47.481280 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:47.510931 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:47.528241 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:47.564206 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:47.660748 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:47.674094 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:47.712803 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:47.752713 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:47.755831 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
I20240502 02:02:47.799618 1628 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:47.824751 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:47.846872 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:47.983723 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 02:02:47.984500 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:47.992427 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:47.998646 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:48.031397 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:48.073725 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:48.088963 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:48.221757 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:48.274582 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:48.371554 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:48.378381 1628 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:48.445241 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:48.481781 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:48.507668 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:48.610266 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:48.620995 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:48.625547 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:48.676710 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:48.695871 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:48.708909 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:48.753948 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:48.769340 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:48.819980 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:48.877956 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:48.890415 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:48.913990 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:48.932909 1731 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:49.018823 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:49.066936 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:49.094625 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:49.154489 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:49.203752 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:49.313339 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:49.373658 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:02:49.453542 1991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:49.622411 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:49.674520 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:49.779654 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:49.786388 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:49.847326 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:49.861014 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:49.868441 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:49.905617 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:49.952632 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
I20240502 02:02:49.972967 2094 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:50.000427 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:50.059994 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:50.129107 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:50.137336 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:50.138425 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:50.166101 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:50.226940 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:50.254395 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:50.300422 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:50.335005 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
I20240502 02:02:50.468097 2094 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:50.482108 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:50.535550 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:50.560685 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:50.597123 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:50.674232 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:50.684614 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:50.716251 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 02:02:50.722170 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 99 similar messages]
W20240502 02:02:50.765085 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:50.768106 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:50.783371 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:50.798449 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:50.817286 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:50.857878 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
I20240502 02:02:50.862768 2094 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:51.079178 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:51.113642 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:51.158466 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:51.177305 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:51.193208 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:51.226646 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:51.239847 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:51.243520 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
I20240502 02:02:51.246762 1991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:51.341446 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:51.360239 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:51.375944 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:51.381278 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 92 similar messages]
W20240502 02:02:51.429260 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:51.579799 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 82 similar messages]
W20240502 02:02:51.580566 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:51.589933 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:51.702479 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:51.761843 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:51.769932 1991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:51.865398 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:51.880970 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:51.908793 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:52.009799 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:52.239428 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:52.239528 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:02:52.264837 1991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:52.282747 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 71 similar messages]
W20240502 02:02:52.283468 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:52.292640 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:52.312963 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:52.353338 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 49 similar messages]
W20240502 02:02:52.411804 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:52.449551 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:52.509114 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:52.538678 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:52.625924 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:52.669116 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:52.672818 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:52.705653 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:52.791080 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:52.791385 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
I20240502 02:02:52.823312 2663 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:52.861244 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:52.862902 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:52.879757 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:52.915673 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:52.916318 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:52.964771 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:53.170758 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 02:02:53.184362 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:02:53.282352 1991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:53.286778 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:53.287019 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:53.302208 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:53.339257 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:53.351287 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:53.351641 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:02:53.365303 2350 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:02:53.365440 2350 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:02:53.365952 2350 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:02:53.366278 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:02:53.366441 15329 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:02:53.366941 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:53.367030 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:53.367081 15396 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:02:53.367192 2350 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:02:53.391904 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:53.408613 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:53.511350 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:53.571648 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:53.679783 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:53.685940 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:53.693272 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:53.736673 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
I20240502 02:02:53.778010 1991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:53.788302 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:53.828373 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:53.859382 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:53.908390 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:53.956809 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:53.968140 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:54.012431 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:54.232169 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:54.267117 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:54.295063 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
I20240502 02:02:54.356037 1991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:54.362948 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:54.380947 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:54.420112 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:54.559533 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:54.574983 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:54.677235 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:54.705910 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:54.716312 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:54.808354 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:54.810004 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:02:54.852258 1991 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:54.917086 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:54.943969 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:54.952360 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:54.961228 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 02:02:55.094887 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:55.109094 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:55.222118 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:55.225113 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:55.237247 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
I20240502 02:02:55.275167 2972 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:55.305634 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:55.328778 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:55.356477 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:55.427834 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:55.431512 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:55.435166 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:55.639154 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:55.652258 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:55.705077 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:55.792233 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:55.835108 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:55.835108 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
I20240502 02:02:55.850157 3284 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:55.864730 14994 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 98 similar messages]
W20240502 02:02:55.898490 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:55.934325 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:55.944532 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:55.950062 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:56.030949 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:56.055717 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:56.208230 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:56.253229 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:56.254434 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:02:56.298450 3284 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:56.311164 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:56.369411 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:56.399089 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 86 similar messages]
W20240502 02:02:56.404155 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:56.429241 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:56.463735 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:56.510532 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:56.629554 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:56.632166 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 83 similar messages]
W20240502 02:02:56.691452 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:56.757853 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:56.774773 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:56.797435 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:56.836822 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:02:56.845813 3284 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:56.850356 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:56.983108 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:57.039419 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:57.086930 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:57.146664 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:57.155683 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:57.174005 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:57.184237 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:57.248498 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:57.285661 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
I20240502 02:02:57.288928 3284 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:57.300431 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:57.411034 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 02:02:57.446837 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 71 similar messages]
W20240502 02:02:57.455091 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:57.472252 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:57.568629 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:57.633266 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
I20240502 02:02:57.666637 3284 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:57.785519 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:57.821715 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:57.847615 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:57.883854 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:57.975628 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:58.010617 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:58.036242 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:58.043128 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:58.058480 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:58.136250 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
I20240502 02:02:58.249929 3284 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:58.258985 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:58.302726 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:58.311164 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:58.324034 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:58.347684 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:58.394176 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:58.395097 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:58.400347 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:58.453748 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 02:02:58.570139 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:58.602710 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:58.722836 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:58.736656 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:58.793052 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:58.811110 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:58.815958 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:58.820127 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:58.855903 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
I20240502 02:02:58.856021 3792 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:58.916008 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:58.919526 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:59.072403 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:59.200580 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:59.226758 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:59.280759 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:59.374130 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:02:59.389246 3792 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:59.418231 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:02:59.442687 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:59.444274 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:59.470106 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:59.530913 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:02:59.532013 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:02:59.667898 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:59.802536 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:02:59.803522 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:02:59.803555 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:02:59.842959 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:02:59.852501 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:02:59.859290 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:02:59.900910 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
I20240502 02:02:59.907011 3792 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:02:59.982049 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:02:59.996980 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:00.285359 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:00.290285 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
I20240502 02:03:00.371670 4111 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:00.389636 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:00.415109 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:00.452263 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:00.461884 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:00.481189 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:00.526389 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:00.598362 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:00.730638 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:00.781828 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:00.803720 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:00.869701 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:00.872818 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 104 similar messages]
W20240502 02:03:00.918081 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:03:00.922188 4111 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:00.933670 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:00.996852 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:01.043306 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 02:03:01.089426 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:01.091524 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:01.114123 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:01.131456 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:01.181728 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:01.206642 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:01.321789 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
I20240502 02:03:01.376363 4111 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:01.403550 15266 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 02:03:01.403781 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:01.408442 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:01.421420 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:01.460530 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:01.467569 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:01.507107 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:01.618572 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:01.619256 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:01.625818 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:01.679777 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 78 similar messages]
W20240502 02:03:01.704900 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:01.725282 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:01.777665 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:01.824527 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
I20240502 02:03:01.844053 4318 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:01.993850 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:02.064504 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:02.088395 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:02.092931 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:02.149159 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:02.179894 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:02.211118 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:02.242551 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:02.277213 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:02.312443 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
I20240502 02:03:02.380136 4318 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:02.403465 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:02.449250 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:02.458483 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 02:03:02.472435 16773 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 02:03:02.472954 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:02.529366 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:02.583575 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:02.624841 4674 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:03:02.624971 4674 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:03:02.625449 4674 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:03:02.625766 15328 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:03:02.625934 15328 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:03:02.626583 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:02.626680 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:02.626741 15399 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:03:02.626853 4674 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:03:02.798168 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
I20240502 02:03:02.874750 4318 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:02.883651 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:02.906064 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:02.937660 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:02.962649 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:02.988037 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:03.075578 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:03.091305 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:03.101718 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:03.292064 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
I20240502 02:03:03.366649 4318 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:03.373149 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:03.386101 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:03.386723 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 02:03:03.421098 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:03.459849 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:03.546573 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:03.597863 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:03.629499 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:03.660151 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:03.660903 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:03.689918 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:03.700222 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:03.758435 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:03.782094 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:03.849833 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:03.895604 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:03.917868 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:03.952375 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
I20240502 02:03:03.976250 4318 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:04.071693 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:04.080590 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:04.122521 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:04.215237 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:04.285705 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:04.304265 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:04.344805 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
I20240502 02:03:04.403563 4623 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:04.434134 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:04.470566 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:04.472324 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:04.476600 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:04.484205 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:04.486097 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:04.652524 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:04.654944 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:04.688413 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:04.821668 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:04.846810 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:04.879987 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:04.896311 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:04.922632 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:03:05.009596 5136 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:05.045629 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:05.255527 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:05.284958 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:05.320976 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:05.347467 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:05.349912 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:05.356245 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
I20240502 02:03:05.422196 4623 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:05.481467 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:05.510752 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:05.539310 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:05.697104 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:05.712206 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:05.731294 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:05.735170 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:05.735564 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:05.893231 14992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 95 similar messages]
W20240502 02:03:05.893811 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:05.937248 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:05.944186 4623 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:05.983011 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 02:03:06.069615 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:06.075074 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:06.121160 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:06.137943 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:06.183300 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:06.241690 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:06.254658 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:06.327507 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:06.338377 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:06.345547 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:06.361222 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:06.424921 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 90 similar messages]
W20240502 02:03:06.444566 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:06.460937 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
I20240502 02:03:06.543771 4623 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:06.552101 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:06.604147 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:06.635700 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:06.678324 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:06.689646 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:06.692523 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 82 similar messages]
W20240502 02:03:06.792742 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:06.841226 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:06.879667 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:06.935329 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:06.981755 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:06.985442 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:07.011097 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:07.029817 5136 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:07.096958 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:07.159024 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:07.220913 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:07.296345 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:07.337538 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:07.373106 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:07.404084 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:07.412431 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:07.433385 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:07.464964 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:07.481312 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 49 similar messages]
W20240502 02:03:07.492110 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:07.529891 15127 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 71 similar messages]
I20240502 02:03:07.570403 5646 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:07.639976 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:07.816314 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:07.822161 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:07.836236 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:07.864696 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:07.904004 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:03:07.966953 5646 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:08.033972 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:08.075228 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:08.100329 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:08.184094 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:08.197949 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:08.205260 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:08.242441 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:08.377272 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:08.412452 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:08.428695 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:08.460021 5136 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:08.592425 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:08.643187 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 02:03:08.681558 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:08.689677 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:08.760468 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:08.771349 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:08.783643 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:08.800876 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:08.808374 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:08.823415 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:08.894271 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:08.900352 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:08.929442 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:08.963660 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:09.017910 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:09.022907 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:09.026151 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
I20240502 02:03:09.039356 5136 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:09.055191 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:09.188030 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:09.282024 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:09.293716 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:09.354912 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:09.362919 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:09.477429 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:09.545126 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
I20240502 02:03:09.576331 5136 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:09.608426 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:09.659307 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:09.741041 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:09.820447 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:09.845402 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:09.886997 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:09.978964 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:10.011386 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:10.012084 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:10.042909 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:10.072098 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
I20240502 02:03:10.130137 5957 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:10.195189 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:10.374445 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:10.440997 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:10.443727 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:10.462291 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:10.499279 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:10.604107 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
I20240502 02:03:10.643936 6313 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:10.651295 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:10.651294 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:10.658739 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:10.700495 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:10.721625 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:10.824366 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:10.847771 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:10.956411 14992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 101 similar messages]
I20240502 02:03:10.983392 4216 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:03:10.983526 4216 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:03:10.984014 4216 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:03:10.984432 15328 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:03:10.984638 15328 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:03:10.985459 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:10.985805 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:10.985810 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:03:10.985865 15396 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:03:10.986001 4216 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:03:10.993557 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:11.001204 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:11.040057 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:11.171222 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:11.187889 6313 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:11.240515 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:11.266348 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:11.302618 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:11.315186 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 02:03:11.317394 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:11.336753 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:11.350732 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:11.363428 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:11.387622 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:11.407384 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:11.434253 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:11.478848 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 02:03:11.497480 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:11.581410 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:11.649514 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:11.660147 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:11.680442 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
I20240502 02:03:11.714180 6419 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:11.723304 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 83 similar messages]
W20240502 02:03:11.723887 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:11.824857 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:11.972626 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:12.089442 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:12.089812 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:12.177814 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:12.211963 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:12.229416 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:12.263200 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:12.320328 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
I20240502 02:03:12.331554 6419 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:12.406409 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:12.518080 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:12.521870 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 48 similar messages]
W20240502 02:03:12.530694 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 70 similar messages]
W20240502 02:03:12.543351 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:12.611641 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:12.613679 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:12.633675 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:12.732821 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:12.795233 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:12.825258 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
I20240502 02:03:12.839754 6419 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:12.907840 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:12.979182 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:12.984943 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:13.066807 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:13.136832 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:13.153887 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:13.174379 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:13.215744 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:13.237344 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:13.291414 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
I20240502 02:03:13.341454 6829 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:13.356566 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:13.483304 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:13.543453 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:13.552332 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:13.624141 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:13.639298 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:13.707943 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:13.768946 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:13.802758 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:13.816370 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
I20240502 02:03:13.840646 6829 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:13.858279 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:13.907312 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:13.907716 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:13.919476 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:13.972949 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:13.986212 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:14.005187 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:14.019935 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 02:03:14.026551 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:14.055797 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:14.084079 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:14.161407 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:14.291488 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:14.353140 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:14.443040 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:14.447685 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:03:14.465387 7082 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:14.544685 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:14.580026 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:14.656062 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:14.753533 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:14.770149 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:14.813421 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:14.897797 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:14.909001 14901 catalog_manager.cc:5014] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 3f938c5af51a43c9a23b753c4991e41d with cas_config_opid_index 15: ChangeConfig:ADD_PEER:NON_VOTER failed with leader 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323) due to error UNKNOWN_ERROR; will retry: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }.
Pending config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
W20240502 02:03:14.909543 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
I20240502 02:03:14.933100 7082 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:15.065353 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:15.079197 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:15.132587 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:15.133157 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:15.184953 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:15.257016 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:15.257017 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:15.417251 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:15.431000 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:15.488546 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:03:15.512984 7082 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:15.563530 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:15.589915 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:15.695612 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:15.785995 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:15.790529 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:15.856894 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:15.883922 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:15.939463 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:15.942116 7082 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:15.984488 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:15.990051 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 100 similar messages]
W20240502 02:03:16.059942 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:16.067692 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:16.138159 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:16.193133 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:16.276225 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:16.291566 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:16.311874 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:16.345165 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:16.356204 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:16.375276 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:16.397810 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
I20240502 02:03:16.405653 7082 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:16.444228 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:16.467387 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:16.493404 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 90 similar messages]
W20240502 02:03:16.508765 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:16.570345 14901 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 2 similar messages]
W20240502 02:03:16.571108 14901 catalog_manager.cc:4573] TS d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 21e2e55d1a0d4b6eb18b4ade9468607f: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:16.578627 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 02:03:16.630375 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:16.651995 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:16.653295 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:16.757398 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 78 similar messages]
W20240502 02:03:16.757968 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:16.816589 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:16.822575 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:16.891855 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:16.929172 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:16.934520 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
I20240502 02:03:16.988642 7082 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:17.014279 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:17.065964 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:17.119129 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:17.216202 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:17.253157 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:17.326436 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:17.329715 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:17.413398 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:17.482278 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:17.488366 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:17.492359 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:17.541728 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 51 similar messages]
I20240502 02:03:17.594703 7598 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:17.598212 15127 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 73 similar messages]
W20240502 02:03:17.611804 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:17.694773 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:17.816785 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:17.859094 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:17.905946 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:17.906823 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:18.005559 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:18.035130 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:18.145684 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:18.185690 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:18.205307 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
I20240502 02:03:18.213464 7598 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:18.252718 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:18.266939 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:18.287730 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
I20240502 02:03:18.318709 4216 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:03:18.318830 4216 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:03:18.319306 4216 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:03:18.319612 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:03:18.319789 15329 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:03:18.320348 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:18.320442 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:03:18.320502 15396 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:03:18.320647 4216 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:03:18.359158 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:18.396778 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:18.428382 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:18.491196 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:18.571187 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:18.626379 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:18.683091 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
I20240502 02:03:18.788345 7849 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:18.794318 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:18.807744 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:18.823822 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:18.847182 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:18.865826 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:18.866503 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:18.878073 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:18.956070 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:19.024556 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 02:03:19.055159 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:19.090867 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:19.099020 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:19.196543 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:19.280547 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:19.300544 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:19.323804 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:19.358939 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
I20240502 02:03:19.383404 7849 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:19.449434 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:19.511284 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:19.531921 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:19.595556 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:19.685021 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:19.699154 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:19.704492 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:19.762212 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:19.797518 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:19.887689 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:19.900677 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:19.905798 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
I20240502 02:03:19.925951 8176 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:20.034458 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:20.055238 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:20.102855 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:20.110082 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:20.286660 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:20.289553 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:20.395656 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:20.403012 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
I20240502 02:03:20.481285 8292 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:20.510852 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:20.513265 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:20.690809 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:20.715523 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:20.716563 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:20.777130 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:20.788187 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:20.817874 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:20.848002 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:20.885577 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:20.893631 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:20.967363 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:21.002573 14992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 102 similar messages]
W20240502 02:03:21.003279 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
I20240502 02:03:21.043076 8292 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:21.096453 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:21.141335 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:21.290910 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:21.315569 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:21.337863 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:21.339223 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:21.389628 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:21.401455 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:21.413210 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 02:03:21.420706 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:21.447417 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:21.508097 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:21.517393 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:21.564738 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 90 similar messages]
I20240502 02:03:21.664286 8292 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:21.704156 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:21.767621 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:21.836356 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 83 similar messages]
W20240502 02:03:21.837038 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:21.853410 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:21.857590 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:21.875341 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:21.944026 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:22.042060 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:22.102185 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:22.147286 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:22.160588 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:22.172822 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
I20240502 02:03:22.213346 8292 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:22.250839 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:22.290948 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:22.438745 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:22.441435 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:22.516889 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:22.574056 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 02:03:22.574664 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:22.614681 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
I20240502 02:03:22.633464 8292 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:22.659840 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 70 similar messages]
W20240502 02:03:22.681412 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:22.693890 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:22.757592 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:22.845717 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:22.883752 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:22.936810 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:23.041584 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:23.075841 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:23.118348 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:23.162983 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:23.229152 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:23.231359 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
I20240502 02:03:23.241926 8852 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:23.265414 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:23.289342 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:23.447154 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:23.461957 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:23.531862 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:23.586524 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:23.609395 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:23.620456 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
I20240502 02:03:23.689826 8292 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:23.741945 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:23.773561 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:23.784060 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:23.794823 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:23.799214 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:23.805955 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:23.822978 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 02:03:23.861570 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:23.982136 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:24.020197 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:24.059993 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:24.072597 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:24.094437 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:03:24.184450 8852 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:24.207127 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:24.263774 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:24.295305 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:24.308328 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:24.399443 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:24.473032 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:24.530892 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:24.551419 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:24.704119 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:24.715512 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
I20240502 02:03:24.715677 8852 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:24.719627 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:24.727713 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:24.783171 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:24.818125 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:24.994915 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:25.124239 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:25.174532 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:25.176359 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:25.213572 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:25.249261 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
I20240502 02:03:25.312649 8852 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:25.324854 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:25.334100 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:25.509246 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:25.516285 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:25.544765 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:25.632189 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:25.647446 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:25.707597 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:25.731649 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:25.779616 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:25.783320 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:25.865099 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:25.902434 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
I20240502 02:03:25.922577 8852 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:25.971916 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:26.052379 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 99 similar messages]
W20240502 02:03:26.052994 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:26.171046 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:26.191795 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:26.191817 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:26.192791 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:26.195981 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:26.212111 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
I20240502 02:03:26.298782 9423 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:26.338873 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:26.362242 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:26.369203 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:26.383493 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:26.388656 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 02:03:26.403950 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:26.431644 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:26.585295 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 88 similar messages]
W20240502 02:03:26.606072 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:26.659513 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:26.702208 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:26.703356 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
I20240502 02:03:26.774705 9423 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:26.844099 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 02:03:26.863664 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:26.877490 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:26.896998 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:26.934234 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:26.939110 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:27.040633 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:27.057467 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:27.103304 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:27.139640 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
I20240502 02:03:27.200773 9423 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:27.244216 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:27.295975 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:27.348451 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:27.552186 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:27.589998 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 02:03:27.634155 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:27.641201 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:27.653894 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 02:03:27.718358 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:27.726742 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 02:03:27.755882 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:03:27.792886 9423 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:27.845446 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:27.921408 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:27.969537 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:28.047912 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:28.088354 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:28.128175 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:28.142246 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:28.151441 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:28.159075 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:28.160187 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:28.209481 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:28.239113 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
I20240502 02:03:28.268885 9782 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:28.434319 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:28.485874 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:28.512586 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:28.545909 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:28.574188 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:28.589010 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:28.629141 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:28.641769 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:28.686534 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:28.706638 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:28.735680 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:28.736312 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
I20240502 02:03:28.823555 9988 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:28.839689 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:28.869907 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:28.905014 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
I20240502 02:03:28.936623 9679 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:03:28.936767 9679 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:03:28.937358 9679 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:03:28.937688 15328 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:03:28.937842 15328 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:03:28.938395 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:28.938494 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:28.938547 15399 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:03:28.938710 9679 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:03:28.946313 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:29.031960 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:29.120901 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:29.137513 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:29.177506 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:29.178212 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:29.259464 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:29.303388 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:29.336010 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
I20240502 02:03:29.342176 9782 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:29.364310 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:29.368659 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:29.378007 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:29.492837 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:29.496654 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:29.532339 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:29.607918 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:29.697399 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:29.748843 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:03:29.846796 10193 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:29.851864 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:29.860281 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:29.904532 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:29.927305 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:29.939476 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:30.118716 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:30.173167 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:30.184008 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 02:03:30.287770 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:30.364207 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:30.374780 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:30.422222 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
I20240502 02:03:30.461205 10346 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:30.551229 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:30.627744 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:30.646915 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:30.651686 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:30.782020 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:30.825058 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:30.845531 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:30.851920 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:30.963064 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:30.971668 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:03:30.988293 10346 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:31.034278 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:31.051345 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:31.071332 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:31.088954 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:31.130906 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 105 similar messages]
W20240502 02:03:31.150843 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:31.182799 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:31.266175 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:31.330622 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:31.370152 10346 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:31.391219 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:31.396423 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:31.445250 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:31.494854 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:31.551721 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:31.591619 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 88 similar messages]
W20240502 02:03:31.658109 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:31.682521 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:31.793387 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:31.795794 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:31.836215 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
I20240502 02:03:31.880451 10449 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:31.892182 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:31.916611 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:31.937500 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:31.939779 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:31.944329 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:32.011147 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:32.022946 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 84 similar messages]
W20240502 02:03:32.101285 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:32.125478 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:32.164062 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:32.165550 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:32.180573 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:32.318123 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
I20240502 02:03:32.457425 10346 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:32.473901 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:32.504729 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:32.599109 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 49 similar messages]
W20240502 02:03:32.609128 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:32.655896 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:32.739454 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:32.754218 15127 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 02:03:32.755621 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:32.837641 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:32.842797 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:32.913930 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
I20240502 02:03:33.009342 10346 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:33.021795 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:33.058739 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:33.059206 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:33.091548 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:33.099426 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:33.131436 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:33.156067 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:33.317687 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:33.438526 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:03:33.511482 10960 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:33.581138 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:33.608111 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:33.641002 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:33.644877 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:33.685599 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:33.718149 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:33.725036 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:33.729387 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:33.738026 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:33.778105 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:33.807437 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:33.844213 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:33.856829 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:33.868911 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:33.993091 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:03:34.094534 10346 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:34.136693 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:34.249006 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:34.262990 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:34.283663 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:34.285809 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:34.426131 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:34.430608 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:34.434207 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:34.441777 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:34.470155 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:34.531235 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
I20240502 02:03:34.667488 11173 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:34.684832 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 02:03:34.691684 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:34.731581 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:34.735868 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:34.803704 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:34.810544 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:34.826848 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:34.869940 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:35.067716 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:35.069083 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:35.192533 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:35.194378 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:35.203752 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:03:35.204303 11312 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:35.234215 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:35.326448 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:35.369922 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:35.522665 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:35.653115 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:35.659513 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:35.669474 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:35.692433 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:35.706447 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
I20240502 02:03:35.750005 11431 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:35.762799 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:35.831171 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:35.874780 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:36.002616 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:36.068156 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:36.122694 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:36.154495 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:36.161870 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:36.198527 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:36.221467 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:36.224982 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 96 similar messages]
W20240502 02:03:36.239233 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:36.241063 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:36.255136 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:36.301100 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:36.325534 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
I20240502 02:03:36.371729 11312 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:36.454430 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:36.526677 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:36.532982 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:36.557781 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:36.627009 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 89 similar messages]
W20240502 02:03:36.655303 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:36.662173 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:36.678093 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:36.683228 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:36.720741 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:36.773177 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:36.796555 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:36.873044 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
I20240502 02:03:36.958357 11649 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:37.044160 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:37.050935 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:37.063426 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:37.109995 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 02:03:37.132573 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 79 similar messages]
I20240502 02:03:37.143085 9679 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:03:37.143184 9679 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:03:37.143618 9679 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:03:37.143921 15328 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:03:37.144068 15328 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:03:37.144762 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:37.144862 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:37.144902 15399 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:03:37.145043 9679 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:03:37.180541 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:37.194648 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:37.326435 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:37.389861 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
I20240502 02:03:37.421597 11649 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:37.445572 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:37.549474 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:37.557463 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:37.619423 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:37.622877 16774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 49 similar messages]
W20240502 02:03:37.667321 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:37.702857 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:37.743077 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:37.768420 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:37.825978 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:37.844508 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:37.865386 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 71 similar messages]
I20240502 02:03:37.961326 11649 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:38.004731 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:38.021059 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:38.061652 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:38.089661 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:38.090217 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:38.146409 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:38.201243 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:38.234033 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:38.273551 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:38.294250 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:38.452327 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:38.528749 12010 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:38.531107 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:38.609906 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:38.611927 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:38.690464 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:38.703836 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:38.737905 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:38.830386 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:38.892212 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:38.893330 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:38.914808 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:03:38.924100 12010 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:38.932674 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:38.999075 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:39.027796 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:39.034686 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:39.116437 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:39.117287 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:39.188484 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:39.269356 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:39.295740 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:39.314954 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:39.343853 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:39.366262 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:39.429694 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:39.445770 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
I20240502 02:03:39.469061 12165 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:39.502563 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 02:03:39.507057 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:39.507151 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:39.568703 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:39.686718 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:39.920074 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:39.923696 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:39.968073 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:39.986502 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:39.997283 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
I20240502 02:03:40.041402 12165 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:40.109742 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:40.196015 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:40.281347 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:40.364071 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:40.373406 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:40.382881 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:40.457787 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:40.473201 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
I20240502 02:03:40.483206 12165 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:40.499249 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:40.586040 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:40.607247 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:40.653591 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:40.666257 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:40.707370 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:40.845584 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:40.866374 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
I20240502 02:03:40.874673 12165 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:40.945220 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:41.000633 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:41.145186 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:41.188428 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:41.213207 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:41.237283 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 100 similar messages]
W20240502 02:03:41.254906 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:41.288717 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:41.291433 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:41.305528 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
I20240502 02:03:41.328714 12165 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:41.414216 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:41.454926 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:41.542786 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:41.611349 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:41.651098 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:41.683710 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:41.740944 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 02:03:41.741570 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:41.776041 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:41.796631 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:41.808713 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:41.847157 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
I20240502 02:03:41.894217 12165 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:41.903725 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:41.922390 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 02:03:41.952634 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:42.065811 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:42.144910 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:42.189391 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 02:03:42.206422 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:42.208127 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:42.229063 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:42.359702 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:03:42.385440 12321 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:42.399690 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:42.457392 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:42.459995 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:42.578032 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:42.687317 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:42.717679 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:42.728569 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:42.785392 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 49 similar messages]
W20240502 02:03:42.786157 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:42.800246 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:42.802681 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:42.821226 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:42.828356 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:42.854755 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:03:42.912478 12321 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:42.927776 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:42.928386 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 70 similar messages]
W20240502 02:03:43.036407 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:43.060336 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:43.077100 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:43.156064 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:43.344118 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:43.367389 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:43.421487 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:43.429864 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:43.468468 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
I20240502 02:03:43.474582 13002 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:43.528913 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:43.542049 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:43.630295 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:43.700651 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:43.727022 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:43.760334 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:43.836270 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:43.870746 12321 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:43.871731 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:43.873333 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:44.008148 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:44.034740 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:44.093873 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:44.147071 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:44.222391 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:44.262249 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
I20240502 02:03:44.264940 13002 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:44.343137 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:44.374415 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:44.377061 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:44.377095 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:44.446162 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:44.460059 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:44.533885 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:44.571768 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 02:03:44.648058 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:44.649456 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:44.661738 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:44.685861 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:44.738107 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:44.862607 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:44.873275 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:44.887264 13002 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:44.905916 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:45.024940 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:45.160008 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:45.207760 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:45.252929 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:45.309653 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:45.329030 13314 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:45.345893 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:45.438191 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:45.451436 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:45.484285 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:45.507925 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:45.514649 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:45.514744 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:45.515157 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:45.609242 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:45.729914 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:45.818995 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:45.823909 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
I20240502 02:03:45.859597 13002 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:45.965188 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:46.051682 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:46.114439 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:46.115909 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:46.175231 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:46.205649 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:46.232798 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:46.240617 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:46.243386 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 99 similar messages]
W20240502 02:03:46.244009 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:46.378453 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:46.423471 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:46.424481 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
I20240502 02:03:46.435165 13630 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:46.483841 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:46.498780 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:46.504845 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:46.625006 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:46.682776 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:46.683115 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:46.699245 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:46.767776 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:46.795120 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:46.859251 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 90 similar messages]
W20240502 02:03:46.859819 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 02:03:46.973901 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:46.980294 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:03:46.986330 13733 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:46.986864 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:47.163110 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:47.174321 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:47.200157 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:47.243745 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 02:03:47.269992 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:47.335480 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:47.356832 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:47.426143 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:47.432536 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
I20240502 02:03:47.436607 13630 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:47.503325 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:47.537355 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:47.596256 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:47.645026 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:47.654562 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:47.741204 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:47.863708 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:47.886564 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:47.910730 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 50 similar messages]
W20240502 02:03:47.912956 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:47.959987 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
I20240502 02:03:47.987800 13733 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:47.996073 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:48.027704 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:48.061369 15127 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 70 similar messages]
W20240502 02:03:48.061947 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:48.230345 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:48.247342 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:48.254350 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:48.488149 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:48.577479 13733 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:48.604593 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:48.643213 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:48.645783 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:48.696667 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:48.706521 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:48.708441 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:48.777158 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:48.791688 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:48.837332 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:48.854480 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:48.948760 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:48.959987 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:48.966249 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:49.022492 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:49.026224 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:03:49.039258 13733 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:49.067813 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:49.186578 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:49.249428 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:49.271967 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:49.332139 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:49.367347 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:49.399214 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 02:03:49.412052 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:49.441576 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:49.544592 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
I20240502 02:03:49.547292 13733 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:49.614284 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:49.622898 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:49.692533 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:49.797526 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:49.838094 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:49.892218 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:49.905489 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:49.913182 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:49.942694 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:50.000880 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:50.007395 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:50.059504 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:03:50.110071 14357 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:50.120276 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:50.150492 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:50.257488 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:50.373031 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:50.431710 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:50.442016 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:50.442451 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:50.460726 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:50.463238 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:50.535740 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:50.585434 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:03:50.590066 13733 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 02:03:50.746297 13471 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:03:50.746451 13471 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:03:50.747051 13471 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:03:50.747437 15328 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:03:50.747604 15328 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:03:50.748191 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:50.748342 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:03:50.748399 15396 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:03:50.748518 13471 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:03:50.795490 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:50.906733 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:50.938525 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:50.939769 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
I20240502 02:03:51.026896 14357 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:51.084041 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:51.151732 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:51.158705 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:51.191985 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:51.215494 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:51.230177 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:51.294199 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:51.311509 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:51.328019 14994 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 101 similar messages]
W20240502 02:03:51.329823 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:51.343367 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:51.374094 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:51.411712 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
I20240502 02:03:51.490072 14357 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:51.497118 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:51.517060 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:51.524564 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:51.587554 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:51.631362 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:51.721251 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:51.874486 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:51.910236 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 02:03:51.911186 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 02:03:51.913069 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:51.920506 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:51.960983 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:52.019805 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:52.053972 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:52.064903 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:52.077288 14357 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:52.107440 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:52.222148 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:52.239780 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:52.265959 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 02:03:52.341598 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:52.376921 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:52.427222 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
I20240502 02:03:52.632203 14889 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:52.637576 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:52.641117 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:52.645313 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:52.650710 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:52.651583 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:52.656091 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:52.799405 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:52.834710 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:52.841933 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:52.910346 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:52.912444 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:52.917969 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 53 similar messages]
W20240502 02:03:52.955819 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:53.112390 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 02:03:53.146953 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:53.235877 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:53.246953 15587 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:53.301367 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:53.354388 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:53.355684 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:53.376375 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:53.409651 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:53.481909 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:53.510339 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:53.692939 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:53.699632 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:53.715816 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:53.726548 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:53.752746 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:53.789009 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:53.792745 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
I20240502 02:03:53.806581 15587 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:53.873065 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:53.908162 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:53.920691 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:53.942250 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:54.035452 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:54.052558 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:54.156903 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
I20240502 02:03:54.215330 15587 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:54.277047 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:54.336560 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:54.348165 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 02:03:54.427789 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:54.452353 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:54.457351 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:54.488039 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:54.513396 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:54.599292 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:54.645066 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:54.685616 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
I20240502 02:03:54.739210 15587 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:54.888140 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:54.898414 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:54.901196 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:54.967875 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:54.987473 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:55.064769 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:55.081295 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
I20240502 02:03:55.175256 15937 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:55.193848 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:55.273985 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:55.297780 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:55.338310 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:55.374368 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:55.395471 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:55.453377 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:55.476527 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:55.485426 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:55.504179 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:03:55.731328 16125 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:55.830699 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:55.841347 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:55.849884 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
I20240502 02:03:55.863320 13471 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:03:55.863454 13471 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:03:55.863929 13471 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:03:55.864262 15329 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "f77c2a969bde45b49ac1bbbe242ff171" is_pre_election: true
I20240502 02:03:55.864410 15329 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b66a8940fe54e21bec1b2db25b2fed5 in term 2.
W20240502 02:03:55.864965 15396 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:55.865414 15399 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:03:55.865483 15399 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:03:55.865612 13471 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:03:55.890018 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:55.943254 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:55.949569 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:55.977319 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:56.043650 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:56.092698 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:56.133549 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:56.222812 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:56.228602 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:56.262084 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:56.264622 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:56.283473 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:56.308230 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
I20240502 02:03:56.324560 16255 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:56.335041 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 96 similar messages]
W20240502 02:03:56.337359 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:56.440779 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:56.503573 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:56.507134 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:56.542944 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:56.554172 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:56.565722 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
I20240502 02:03:56.788575 16255 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:56.805809 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:56.813428 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 02:03:56.834403 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:56.856212 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:56.874148 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:56.967478 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:56.973829 15266 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 02:03:56.976384 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:57.000326 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:57.013293 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:57.170616 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:57.209542 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:57.240474 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:57.265182 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:03:57.279016 16125 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:57.351873 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 02:03:57.352432 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:57.370106 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:57.489090 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:57.569631 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:57.575357 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:57.609285 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:57.720235 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:57.769855 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:57.771512 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:57.805330 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:57.812054 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:57.813233 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:57.845752 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
I20240502 02:03:57.884271 16569 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:57.944655 16773 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 47 similar messages]
W20240502 02:03:57.945403 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:58.161514 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 69 similar messages]
W20240502 02:03:58.175328 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:58.291502 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:58.293767 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:58.302958 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:58.338099 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:58.347005 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:58.350042 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:58.379899 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:58.400705 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:03:58.459031 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
I20240502 02:03:58.468439 16125 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:58.495960 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:58.699950 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:58.730226 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:58.751966 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:58.810492 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:58.856933 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:58.861392 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:58.876294 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
I20240502 02:03:58.890190 16569 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:58.937991 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:58.956800 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:59.013070 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:59.026943 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:59.043372 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:03:59.075255 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:59.138116 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:59.143333 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 02:03:59.188493 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:59.278508 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:59.311209 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:59.341447 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:03:59.352447 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:03:59.465216 17008 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:03:59.511919 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:59.696384 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:03:59.697177 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:03:59.699234 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:59.700068 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:59.731570 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:03:59.784433 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:03:59.811738 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:03:59.869437 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
I20240502 02:03:59.869665 16569 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:00.045013 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:00.089074 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:00.171506 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:00.186540 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:04:00.205837 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:00.232277 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:04:00.236606 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:04:00.269089 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:00.334116 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
I20240502 02:04:00.353117 16569 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:00.502897 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:00.512373 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:00.577368 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:00.638005 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:04:00.705291 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:00.731720 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:04:00.743722 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:00.751755 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:00.885537 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:00.912765 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
I20240502 02:04:00.924464 16569 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:00.934954 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:01.045616 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:01.089273 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:01.104724 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:01.140405 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:01.198338 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:01.231005 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:04:01.238260 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:04:01.306434 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:01.331385 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:01.367486 14992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 101 similar messages]
W20240502 02:04:01.392915 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:01.429198 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:01.462765 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
I20240502 02:04:01.497299 17437 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:01.530120 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:01.630246 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:01.639204 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 02:04:01.693193 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:01.714053 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:04:01.714061 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:01.765210 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:01.834939 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:01.852222 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:01.854313 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:04:01.900878 17437 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:01.932201 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:01.990092 15263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 90 similar messages]
W20240502 02:04:01.990154 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:02.144366 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:02.149600 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:02.180262 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:02.238099 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:04:02.288585 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:02.319299 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:02.360877 15396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 02:04:02.361560 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:04:02.421859 17437 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:02.454286 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:02.564376 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:02.617141 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:02.618682 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:02.672945 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:02.713768 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:02.748797 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:04:02.805003 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:04:02.861574 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:02.874210 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:02.949949 16773 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 50 similar messages]
I20240502 02:04:03.045094 17437 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:03.099637 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:03.110478 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:03.110899 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:03.135978 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:03.204550 15131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 73 similar messages]
W20240502 02:04:03.233420 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:04:03.288234 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:03.350397 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:03.352913 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:04:03.379371 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:03.417637 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:03.509994 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:03.639338 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
I20240502 02:04:03.664383 17437 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:03.751181 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:03.756919 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:03.772948 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:03.789490 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:04:03.840150 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:04:03.971891 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:04.051847 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:04.064563 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:04.070247 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
I20240502 02:04:04.086095 17909 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:04.163733 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:04.175496 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:04.176254 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:04.178437 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:04.225553 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:04.270473 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:04.273130 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:04.273867 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:04.301285 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:04.377103 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:04.405534 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:04:04.410563 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:04:04.440678 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 02:04:04.453259 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:04.464282 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:04.560603 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
I20240502 02:04:04.565922 17909 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:04.759486 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:04.799196 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:04:05.002511 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:05.030731 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:05.035557 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:04:05.053790 17909 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:05.093737 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:05.112780 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:05.128993 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:05.146770 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:05.148707 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:05.274667 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:05.317826 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:04:05.345678 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:05.399236 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:05.429272 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:05.532622 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:05.547034 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:04:05.600235 18272 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:05.656888 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:05.713976 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:04:05.722180 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:05.792037 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:05.917327 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
I20240502 02:04:05.985093 17909 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:06.035421 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:06.060415 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 02:04:06.105450 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:04:06.121675 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:04:06.173120 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:06.188093 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:06.285516 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:06.292059 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:06.391184 14995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 95 similar messages]
W20240502 02:04:06.418923 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20240502 02:04:06.432684 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:06.455727 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20240502 02:04:06.460714 18272 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:06.530668 15130 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:06.593554 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
I20240502 02:04:06.605950 17540 raft_consensus.cc:482] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:06.606066 17540 raft_consensus.cc:504] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:04:06.606580 17540 leader_election.cc:288] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323), fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207), 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645)
I20240502 02:04:06.606884 15464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "986e7c97e0734e958f364dc5d1167846" candidate_uuid: "f77c2a969bde45b49ac1bbbe242ff171" candidate_term: 3 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: false dest_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" is_pre_election: true
I20240502 02:04:06.607038 15464 raft_consensus.cc:2454] T 986e7c97e0734e958f364dc5d1167846 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f77c2a969bde45b49ac1bbbe242ff171 in term 2.
W20240502 02:04:06.607609 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:06.607944 15266 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:06.608021 15263 leader_election.cc:334] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:04:06.608078 15263 leader_election.cc:302] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 4 voters: 2 yes votes; 2 no votes. yes voters: 7b66a8940fe54e21bec1b2db25b2fed5, f77c2a969bde45b49ac1bbbe242ff171; no voters: 3b8d615c71e44ff887e0f037c3efcff7, fc067ee690bb457e8109ad8e4eb43ab8
I20240502 02:04:06.608198 17540 raft_consensus.cc:2726] T 986e7c97e0734e958f364dc5d1167846 P f77c2a969bde45b49ac1bbbe242ff171 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 02:04:06.625685 15263 consensus_peers.cc:485] T 6db29f2516e1486782be0312c9bcac8b P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20240502 02:04:06.632771 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:06.673327 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:04:06.686486 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:04:06.728096 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:06.783795 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:06.804950 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:06.834105 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:06.963841 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20240502 02:04:06.964740 16771 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:07.000571 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:07.034981 15265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 91 similar messages]
W20240502 02:04:07.035691 15265 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
I20240502 02:04:07.042054 18272 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:07.066697 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:07.098819 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:04:07.141863 15396 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:07.144438 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:07.228081 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:04:07.257390 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20240502 02:04:07.369499 15399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 02:04:07.499665 15131 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20240502 02:04:07.562036 14995 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
I20240502 02:04:07.576243 18582 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:07.582047 14992 consensus_peers.cc:485] T 53edbd87d0d5471ca7626ca2071f75eb P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:07.603446 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:07.651793 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:04:07.680419 14995 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:07.701231 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
W20240502 02:04:07.725646 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Error code: TABLET_NOT_FOUND (6). Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:07.728170 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:07.811681 15131 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:07.956156 16771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111) [suppressed 48 similar messages]
W20240502 02:04:07.960461 15265 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20240502 02:04:08.054996 18582 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
W20240502 02:04:08.060514 15127 consensus_peers.cc:485] T cd00409d47954ee2b7555e7ff09a6e28 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
I20240502 02:04:08.141191 18685 consensus_queue.cc:579] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Leader has been unable to successfully communicate with peer 3b8d615c71e44ff887e0f037c3efcff7 for more than 300 seconds (300.022s)
W20240502 02:04:08.193627 14992 consensus_peers.cc:485] T 1eb15147ff6c4d0b83b12dec736fc576 P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20240502 02:04:08.193778 15398 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
I20240502 02:04:08.209363 17540 consensus_queue.cc:579] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Leader has been unable to successfully communicate with peer d4e5422f66454511ae5a50ec6ab134ad for more than 300 seconds (300.321s)
W20240502 02:04:08.210449 15265 consensus_peers.cc:462] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:04:08.214946 15329 consensus_queue.cc:237] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } }
I20240502 02:04:08.216131 16838 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 02:04:08.216157 15057 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 02:04:08.216501 18845 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 02:04:08.216621 17540 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 02:04:08.218273 18844 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } }
I20240502 02:04:08.218814 16838 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } }
I20240502 02:04:08.219022 15057 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } }
I20240502 02:04:08.220011 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e2f4ca025890492880608c7975650969 with cas_config_opid_index 11: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
W20240502 02:04:08.220409 15130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 70 similar messages]
I20240502 02:04:08.221213 14911 catalog_manager.cc:5484] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 11 to 12, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } }
W20240502 02:04:08.225960 14901 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111) [suppressed 2 similar messages]
W20240502 02:04:08.226800 14901 catalog_manager.cc:4573] TS d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet e2f4ca025890492880608c7975650969: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111)
I20240502 02:04:08.226760 15329 consensus_queue.cc:237] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: true } }
I20240502 02:04:08.227829 16838 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 02:04:08.228058 15057 raft_consensus.cc:1261] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
W20240502 02:04:08.228123 15264 consensus_peers.cc:485] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905): Couldn't send request to peer 26a3d5ced97d4dcc919b80fde53f6a5b. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:08.228422 18845 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 02:04:08.228677 18844 consensus_queue.cc:1035] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 02:04:08.230125 18844 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, NON_VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) added. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: true } } }
I20240502 02:04:08.230615 15057 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, NON_VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) added. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: true } } }
I20240502 02:04:08.230655 16838 raft_consensus.cc:2932] T e2f4ca025890492880608c7975650969 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, NON_VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) added. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: true } } }
I20240502 02:04:08.232684 14900 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet e2f4ca025890492880608c7975650969 with cas_config_opid_index 12: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 02:04:08.233011 14915 catalog_manager.cc:5484] T e2f4ca025890492880608c7975650969 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 12 to 13, NON_VOTER 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: true } } }
W20240502 02:04:08.274247 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
W20240502 02:04:08.312732 16774 consensus_peers.cc:485] T d24986c2ebab46f086e3f1f5a53af26b P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
I20240502 02:04:08.321846 18116 consensus_queue.cc:579] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Leader has been unable to successfully communicate with peer 3b8d615c71e44ff887e0f037c3efcff7 for more than 300 seconds (300.373s)
W20240502 02:04:08.323155 14995 consensus_peers.cc:462] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
I20240502 02:04:08.326897 15057 consensus_queue.cc:237] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } }
I20240502 02:04:08.328042 15194 raft_consensus.cc:1261] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:08.328415 18856 consensus_queue.cc:1035] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240502 02:04:08.328591 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:08.329943 18116 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 02:04:08.331029 15194 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } }
I20240502 02:04:08.331928 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 8f2fb504a49c44edac92fd935555c278 with cas_config_opid_index 7: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:08.332540 14911 catalog_manager.cc:5484] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 7 to 8, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:08.335888 15057 consensus_queue.cc:237] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } }
W20240502 02:04:08.337281 14993 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): Couldn't send request to peer f77c2a969bde45b49ac1bbbe242ff171. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8f2fb504a49c44edac92fd935555c278. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:08.337535 15194 raft_consensus.cc:1261] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
W20240502 02:04:08.337814 14994 consensus_peers.cc:485] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:08.337873 18116 consensus_queue.cc:1035] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:08.339231 18855 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } } }
I20240502 02:04:08.339639 15194 raft_consensus.cc:2932] T 8f2fb504a49c44edac92fd935555c278 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } } }
I20240502 02:04:08.340917 14901 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 8f2fb504a49c44edac92fd935555c278 with cas_config_opid_index 8: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 02:04:08.341437 14911 catalog_manager.cc:5484] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 8 to 9, NON_VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 02:04:08.364498 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:08.368579 16773 consensus_peers.cc:485] T 8ddc239a297a46e48e069b985cb7b350 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 02:04:08.609503 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
I20240502 02:04:08.629679 18582 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 02:04:08.637779 18801 consensus_queue.cc:579] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Leader has been unable to successfully communicate with peer 3b8d615c71e44ff887e0f037c3efcff7 for more than 300 seconds (300.301s)
I20240502 02:04:08.640558 15464 consensus_queue.cc:237] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 02:04:08.641914 15329 raft_consensus.cc:1261] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 02:04:08.642306 18958 consensus_queue.cc:1035] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
W20240502 02:04:08.644341 15396 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20240502 02:04:08.646787 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fc07e430e3e74e3885f263881bd04976. This is attempt 1: this message will repeat every 5th retry.
W20240502 02:04:08.647183 15399 consensus_peers.cc:485] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:08.647771 18801 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 02:04:08.648187 15329 raft_consensus.cc:2932] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 02:04:08.650116 14901 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet fc07e430e3e74e3885f263881bd04976 with cas_config_opid_index 9: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 02:04:08.650576 14911 catalog_manager.cc:5484] T fc07e430e3e74e3885f263881bd04976 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 9 to 10, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
W20240502 02:04:08.659641 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:04:08.705317 18582 consensus_queue.cc:579] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Leader has been unable to successfully communicate with peer d4e5422f66454511ae5a50ec6ab134ad for more than 300 seconds (300.392s)
I20240502 02:04:08.710839 15194 consensus_queue.cc:237] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } }
I20240502 02:04:08.712067 15464 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 02:04:08.712452 18582 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
W20240502 02:04:08.712879 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:08.714278 18957 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 02:04:08.714728 15464 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } }
I20240502 02:04:08.716003 14900 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 5804597cc507483dba622e6ec44e5ead with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:08.717185 14906 catalog_manager.cc:5484] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 9 to 10, NON_VOTER d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70) evicted. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:08.720558 15194 consensus_queue.cc:237] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } }
I20240502 02:04:08.722455 15464 raft_consensus.cc:1261] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer 26a3d5ced97d4dcc919b80fde53f6a5b: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
W20240502 02:04:08.722585 15131 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 02:04:08.722890 15129 consensus_peers.cc:485] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): Couldn't send request to peer f77c2a969bde45b49ac1bbbe242ff171. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:08.722942 18957 consensus_queue.cc:1035] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 02:04:08.724510 18582 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } } }
I20240502 02:04:08.726231 14900 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 5804597cc507483dba622e6ec44e5ead with cas_config_opid_index 10: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 02:04:08.726673 15464 raft_consensus.cc:2932] T 5804597cc507483dba622e6ec44e5ead P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } } }
I20240502 02:04:08.727114 14906 catalog_manager.cc:5484] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 10 to 11, NON_VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) added. New cstate: current_term: 1 leader_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:08.746311 19032 ts_tablet_manager.cc:902] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 02:04:08.746831 19032 tablet_copy_client.cc:308] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 02:04:08.747284 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet e2f4ca025890492880608c7975650969 from peer 26a3d5ced97d4dcc919b80fde53f6a5b ({username='slave'} at 127.7.221.66:60941)
I20240502 02:04:08.747390 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet e2f4ca025890492880608c7975650969 from peer 26a3d5ced97d4dcc919b80fde53f6a5b at {username='slave'} at 127.7.221.66:60941: session id = 26a3d5ced97d4dcc919b80fde53f6a5b-e2f4ca025890492880608c7975650969
I20240502 02:04:08.748831 15351 tablet_copy_source_session.cc:205] T e2f4ca025890492880608c7975650969 P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 5 blocks and 1 log segments
I20240502 02:04:08.749763 19032 ts_tablet_manager.cc:1875] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 02:04:08.751914 19032 ts_tablet_manager.cc:1888] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.7
I20240502 02:04:08.752064 19032 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e2f4ca025890492880608c7975650969. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:08.752929 19032 tablet_copy_client.cc:783] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet copy: Starting download of 5 data blocks...
W20240502 02:04:08.758291 15263 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
I20240502 02:04:08.758628 19032 tablet_copy_client.cc:647] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:08.759915 19032 tablet_copy_client.cc:523] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:08.761112 19032 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap starting.
W20240502 02:04:08.766942 15130 consensus_peers.cc:485] T 81c8fe21ec7545e79450532b43a61ee7 P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
I20240502 02:04:08.771261 19032 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap replayed 1/1 log segments. Stats: ops{read=13 overwritten=0 applied=13 ignored=2} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:08.771473 19032 tablet_bootstrap.cc:492] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: Bootstrap complete.
I20240502 02:04:08.771608 19032 ts_tablet_manager.cc:1367] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent bootstrapping tablet: real 0.011s user 0.009s sys 0.002s
I20240502 02:04:08.771926 19032 raft_consensus.cc:348] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: true } }
I20240502 02:04:08.772018 19032 raft_consensus.cc:729] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 26a3d5ced97d4dcc919b80fde53f6a5b, State: Initialized, Role: LEARNER
I20240502 02:04:08.772083 19032 consensus_queue.cc:260] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 13, Last appended: 1.13, Last appended by leader: 13, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { promote: true } }
I20240502 02:04:08.772323 19032 ts_tablet_manager.cc:1398] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:08.772716 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session 26a3d5ced97d4dcc919b80fde53f6a5b-e2f4ca025890492880608c7975650969 received from {username='slave'} at 127.7.221.66:60941
I20240502 02:04:08.772785 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session 26a3d5ced97d4dcc919b80fde53f6a5b-e2f4ca025890492880608c7975650969 on tablet e2f4ca025890492880608c7975650969 with peer 26a3d5ced97d4dcc919b80fde53f6a5b
I20240502 02:04:08.794392 19058 ts_tablet_manager.cc:902] T 8f2fb504a49c44edac92fd935555c278 P f77c2a969bde45b49ac1bbbe242ff171: Initiating tablet copy from peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123)
I20240502 02:04:08.794889 19058 tablet_copy_client.cc:308] T 8f2fb504a49c44edac92fd935555c278 P f77c2a969bde45b49ac1bbbe242ff171: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:46123
I20240502 02:04:08.795311 15080 tablet_copy_service.cc:140] P ddae2a636423489693c12b7d7d166c09: Received BeginTabletCopySession request for tablet 8f2fb504a49c44edac92fd935555c278 from peer f77c2a969bde45b49ac1bbbe242ff171 ({username='slave'} at 127.7.221.67:39975)
I20240502 02:04:08.795424 15080 tablet_copy_service.cc:161] P ddae2a636423489693c12b7d7d166c09: Beginning new tablet copy session on tablet 8f2fb504a49c44edac92fd935555c278 from peer f77c2a969bde45b49ac1bbbe242ff171 at {username='slave'} at 127.7.221.67:39975: session id = f77c2a969bde45b49ac1bbbe242ff171-8f2fb504a49c44edac92fd935555c278
I20240502 02:04:08.796543 15080 tablet_copy_source_session.cc:205] T 8f2fb504a49c44edac92fd935555c278 P ddae2a636423489693c12b7d7d166c09: Tablet Copy: opened 5 blocks and 1 log segments
I20240502 02:04:08.797461 19058 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8f2fb504a49c44edac92fd935555c278. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:08.799216 19058 tablet_copy_client.cc:783] T 8f2fb504a49c44edac92fd935555c278 P f77c2a969bde45b49ac1bbbe242ff171: tablet copy: Starting download of 5 data blocks...
I20240502 02:04:08.803525 19058 tablet_copy_client.cc:647] T 8f2fb504a49c44edac92fd935555c278 P f77c2a969bde45b49ac1bbbe242ff171: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:08.804793 19058 tablet_copy_client.cc:523] T 8f2fb504a49c44edac92fd935555c278 P f77c2a969bde45b49ac1bbbe242ff171: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:08.806017 19058 tablet_bootstrap.cc:492] T 8f2fb504a49c44edac92fd935555c278 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 02:04:08.812534 19058 tablet_bootstrap.cc:492] T 8f2fb504a49c44edac92fd935555c278 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap replayed 1/1 log segments. Stats: ops{read=9 overwritten=0 applied=9 ignored=2} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:08.812747 19058 tablet_bootstrap.cc:492] T 8f2fb504a49c44edac92fd935555c278 P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap complete.
I20240502 02:04:08.812898 19058 ts_tablet_manager.cc:1367] T 8f2fb504a49c44edac92fd935555c278 P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.001s
I20240502 02:04:08.813206 19058 raft_consensus.cc:348] T 8f2fb504a49c44edac92fd935555c278 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } }
I20240502 02:04:08.813302 19058 raft_consensus.cc:729] T 8f2fb504a49c44edac92fd935555c278 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: LEARNER
I20240502 02:04:08.813390 19058 consensus_queue.cc:260] T 8f2fb504a49c44edac92fd935555c278 P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 9, Last appended: 1.9, Last appended by leader: 9, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } }
I20240502 02:04:08.813637 19058 ts_tablet_manager.cc:1398] T 8f2fb504a49c44edac92fd935555c278 P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:08.814086 15080 tablet_copy_service.cc:342] P ddae2a636423489693c12b7d7d166c09: Request end of tablet copy session f77c2a969bde45b49ac1bbbe242ff171-8f2fb504a49c44edac92fd935555c278 received from {username='slave'} at 127.7.221.67:39975
I20240502 02:04:08.814158 15080 tablet_copy_service.cc:433] P ddae2a636423489693c12b7d7d166c09: ending tablet copy session f77c2a969bde45b49ac1bbbe242ff171-8f2fb504a49c44edac92fd935555c278 on tablet 8f2fb504a49c44edac92fd935555c278 with peer f77c2a969bde45b49ac1bbbe242ff171
W20240502 02:04:08.885458 16771 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 606: this message will repeat every 5th retry.
I20240502 02:04:08.889273 17540 consensus_queue.cc:579] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Leader has been unable to successfully communicate with peer d4e5422f66454511ae5a50ec6ab134ad for more than 300 seconds (300.090s)
I20240502 02:04:08.891999 15329 consensus_queue.cc:237] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } }
I20240502 02:04:08.893302 15194 raft_consensus.cc:1261] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 02:04:08.893714 19064 consensus_queue.cc:1035] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
W20240502 02:04:08.893946 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123): Couldn't send request to peer ddae2a636423489693c12b7d7d166c09. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 30620f4774b8414eb4e27bd5ecb6ecee. This is attempt 1: this message will repeat every 5th retry.
W20240502 02:04:08.894042 15265 consensus_peers.cc:485] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:08.895653 17540 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } } }
I20240502 02:04:08.896193 15194 raft_consensus.cc:2932] T 30620f4774b8414eb4e27bd5ecb6ecee P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } } }
I20240502 02:04:08.897024 14900 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 30620f4774b8414eb4e27bd5ecb6ecee with cas_config_opid_index 10: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 02:04:08.897893 14911 catalog_manager.cc:5484] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 10 to 11, NON_VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:08.949096 15244 maintenance_manager.cc:419] P 26a3d5ced97d4dcc919b80fde53f6a5b: Scheduling UndoDeltaBlockGCOp(e2f4ca025890492880608c7975650969): 4581 bytes on disk
I20240502 02:04:08.949617 15132 maintenance_manager.cc:643] P 26a3d5ced97d4dcc919b80fde53f6a5b: UndoDeltaBlockGCOp(e2f4ca025890492880608c7975650969) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":69,"lbm_reads_lt_1ms":4}
I20240502 02:04:08.975370 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling UndoDeltaBlockGCOp(8f2fb504a49c44edac92fd935555c278): 4834 bytes on disk
I20240502 02:04:08.975955 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: UndoDeltaBlockGCOp(8f2fb504a49c44edac92fd935555c278) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":72,"lbm_reads_lt_1ms":4}
I20240502 02:04:09.028788 18856 consensus_queue.cc:579] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Leader has been unable to successfully communicate with peer 3b8d615c71e44ff887e0f037c3efcff7 for more than 300 seconds (300.166s)
I20240502 02:04:09.031898 15057 consensus_queue.cc:237] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } }
I20240502 02:04:09.033231 16838 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
W20240502 02:04:09.033727 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:09.034063 18856 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 02:04:09.036286 19108 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 02:04:09.036744 16838 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } }
I20240502 02:04:09.037667 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e00d1ed5cde044cb9bdebf787590261a with cas_config_opid_index 10: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:09.039100 14906 catalog_manager.cc:5484] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 10 to 11, NON_VOTER 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72) evicted. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:09.041960 15057 consensus_queue.cc:237] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } }
I20240502 02:04:09.043124 16838 raft_consensus.cc:1261] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
W20240502 02:04:09.043418 14993 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157): Couldn't send request to peer f77c2a969bde45b49ac1bbbe242ff171. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:09.043455 18856 consensus_queue.cc:1035] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
W20240502 02:04:09.043897 14992 consensus_peers.cc:485] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:09.044899 19107 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } } }
I20240502 02:04:09.045351 16838 raft_consensus.cc:2932] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } } }
I20240502 02:04:09.047137 14901 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet e00d1ed5cde044cb9bdebf787590261a with cas_config_opid_index 11: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 02:04:09.047902 14911 catalog_manager.cc:5484] T e00d1ed5cde044cb9bdebf787590261a P 84987ad9e4a54d96af8569e4aaf0ed64 reported cstate change: config changed from index 11 to 12, NON_VOTER f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { replace: true } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } } }
I20240502 02:04:09.057742 18845 consensus_queue.cc:579] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Leader has been unable to successfully communicate with peer d4e5422f66454511ae5a50ec6ab134ad for more than 300 seconds (300.388s)
W20240502 02:04:09.058405 16774 consensus_peers.cc:485] T 856f623374824d94913db0cb9faa25d1 P 84987ad9e4a54d96af8569e4aaf0ed64 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20240502 02:04:09.060142 15329 consensus_queue.cc:237] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 02:04:09.061745 15464 raft_consensus.cc:1261] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Refusing update from remote peer f77c2a969bde45b49ac1bbbe242ff171: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
W20240502 02:04:09.062079 15266 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 490e37fac0844a08a07a4b281bc33bf9. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:09.062141 19064 consensus_queue.cc:1035] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
W20240502 02:04:09.062285 15263 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 02:04:09.062288 15265 consensus_peers.cc:485] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:09.063812 19112 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 02:04:09.064288 15464 raft_consensus.cc:2932] T 490e37fac0844a08a07a4b281bc33bf9 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true replace: true } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 02:04:09.065191 14900 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 490e37fac0844a08a07a4b281bc33bf9 with cas_config_opid_index 8: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 02:04:09.066538 14906 catalog_manager.cc:5484] T 490e37fac0844a08a07a4b281bc33bf9 P f77c2a969bde45b49ac1bbbe242ff171 reported cstate change: config changed from index 8 to 9, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "f77c2a969bde45b49ac1bbbe242ff171" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:09.085608 18958 consensus_queue.cc:579] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Leader has been unable to successfully communicate with peer fc067ee690bb457e8109ad8e4eb43ab8 for more than 300 seconds (300.365s)
W20240502 02:04:09.086836 15396 consensus_peers.cc:462] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111)
I20240502 02:04:09.088166 15464 consensus_queue.cc:237] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } }
I20240502 02:04:09.089584 15057 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 02:04:09.089824 15194 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 02:04:09.089957 18958 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 02:04:09.090246 18958 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 02:04:09.091851 18801 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } }
I20240502 02:04:09.092375 15057 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } }
I20240502 02:04:09.092581 15194 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } }
I20240502 02:04:09.093704 14901 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet bf73e677bcd049869cadf9f8c9cd3434 with cas_config_opid_index 10: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:09.094995 14911 catalog_manager.cc:5484] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b reported cstate change: config changed from index 10 to 11, NON_VOTER fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71) evicted. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } }
I20240502 02:04:09.098305 15464 consensus_queue.cc:237] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 02:04:09.099409 15057 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 02:04:09.099431 15194 raft_consensus.cc:1261] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
W20240502 02:04:09.099601 15399 consensus_peers.cc:485] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bf73e677bcd049869cadf9f8c9cd3434. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:09.099803 18958 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 02:04:09.099992 18961 consensus_queue.cc:1035] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 02:04:09.101351 18958 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 02:04:09.101861 15194 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 02:04:09.102788 14901 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet bf73e677bcd049869cadf9f8c9cd3434 with cas_config_opid_index 11: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 02:04:09.103046 15057 raft_consensus.cc:2932] T bf73e677bcd049869cadf9f8c9cd3434 P ddae2a636423489693c12b7d7d166c09 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
I20240502 02:04:09.103639 14915 catalog_manager.cc:5484] T bf73e677bcd049869cadf9f8c9cd3434 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 11 to 12, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:09.118134 19058 ts_tablet_manager.cc:902] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: Initiating tablet copy from peer 26a3d5ced97d4dcc919b80fde53f6a5b (127.7.221.66:34905)
I20240502 02:04:09.118726 19058 tablet_copy_client.cc:308] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34905
I20240502 02:04:09.119195 15216 tablet_copy_service.cc:140] P 26a3d5ced97d4dcc919b80fde53f6a5b: Received BeginTabletCopySession request for tablet 5804597cc507483dba622e6ec44e5ead from peer f77c2a969bde45b49ac1bbbe242ff171 ({username='slave'} at 127.7.221.67:40065)
I20240502 02:04:09.119287 15216 tablet_copy_service.cc:161] P 26a3d5ced97d4dcc919b80fde53f6a5b: Beginning new tablet copy session on tablet 5804597cc507483dba622e6ec44e5ead from peer f77c2a969bde45b49ac1bbbe242ff171 at {username='slave'} at 127.7.221.67:40065: session id = f77c2a969bde45b49ac1bbbe242ff171-5804597cc507483dba622e6ec44e5ead
I20240502 02:04:09.120555 15216 tablet_copy_source_session.cc:205] T 5804597cc507483dba622e6ec44e5ead P 26a3d5ced97d4dcc919b80fde53f6a5b: Tablet Copy: opened 5 blocks and 1 log segments
I20240502 02:04:09.121372 19058 ts_tablet_manager.cc:1875] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 02:04:09.122005 19010 consensus_queue.cc:579] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Leader has been unable to successfully communicate with peer 3b8d615c71e44ff887e0f037c3efcff7 for more than 300 seconds (300.165s)
I20240502 02:04:09.123638 19058 ts_tablet_manager.cc:1888] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.7
I20240502 02:04:09.123808 19058 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5804597cc507483dba622e6ec44e5ead. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:09.125157 19058 tablet_copy_client.cc:783] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: tablet copy: Starting download of 5 data blocks...
I20240502 02:04:09.125329 15194 raft_consensus.cc:1203] T e2f4ca025890492880608c7975650969 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.12->[1.13-1.13] Dedup: 1.13->[]
I20240502 02:04:09.127425 18961 consensus_queue.cc:579] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Leader has been unable to successfully communicate with peer fc067ee690bb457e8109ad8e4eb43ab8 for more than 300 seconds (300.110s)
I20240502 02:04:09.130052 15464 consensus_queue.cc:237] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } }
I20240502 02:04:09.131673 16838 raft_consensus.cc:1261] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Refusing update from remote peer 7b66a8940fe54e21bec1b2db25b2fed5: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
W20240502 02:04:09.131790 15398 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer ddae2a636423489693c12b7d7d166c09 (127.7.221.65:46123): Couldn't send request to peer ddae2a636423489693c12b7d7d166c09. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:09.132045 18801 consensus_queue.cc:1035] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
W20240502 02:04:09.132201 15396 consensus_peers.cc:485] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer fc067ee690bb457e8109ad8e4eb43ab8 (127.7.221.71:43207): Couldn't send request to peer fc067ee690bb457e8109ad8e4eb43ab8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.71:43207: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:09.133175 19058 tablet_copy_client.cc:647] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:09.133780 18958 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } } }
I20240502 02:04:09.134253 16838 raft_consensus.cc:2932] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 84987ad9e4a54d96af8569e4aaf0ed64 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } } }
I20240502 02:04:09.135210 14901 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532 with cas_config_opid_index 11: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 02:04:09.135318 19058 tablet_copy_client.cc:523] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:09.136758 19058 tablet_bootstrap.cc:492] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap starting.
I20240502 02:04:09.136735 14906 catalog_manager.cc:5484] T fcfc8dc4b90a4a6aa7e4035bbfdb4532 P 7b66a8940fe54e21bec1b2db25b2fed5 reported cstate change: config changed from index 11 to 12, NON_VOTER ddae2a636423489693c12b7d7d166c09 (127.7.221.65) added. New cstate: current_term: 1 leader_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:09.145308 19058 tablet_bootstrap.cc:492] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=2} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:09.145606 19058 tablet_bootstrap.cc:492] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: Bootstrap complete.
I20240502 02:04:09.145758 19058 ts_tablet_manager.cc:1367] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.000s
I20240502 02:04:09.146104 19058 raft_consensus.cc:348] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } }
I20240502 02:04:09.146206 19058 raft_consensus.cc:729] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f77c2a969bde45b49ac1bbbe242ff171, State: Initialized, Role: LEARNER
I20240502 02:04:09.146301 19058 consensus_queue.cc:260] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } attrs { replace: true } } peers { permanent_uuid: "3b8d615c71e44ff887e0f037c3efcff7" member_type: VOTER last_known_addr { host: "127.7.221.72" port: 37645 } attrs { promote: false } } peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 44157 } attrs { promote: true } }
I20240502 02:04:09.146580 19058 ts_tablet_manager.cc:1398] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171: Time spent starting tablet: real 0.001s user 0.003s sys 0.000s
I20240502 02:04:09.147059 15216 tablet_copy_service.cc:342] P 26a3d5ced97d4dcc919b80fde53f6a5b: Request end of tablet copy session f77c2a969bde45b49ac1bbbe242ff171-5804597cc507483dba622e6ec44e5ead received from {username='slave'} at 127.7.221.67:40065
I20240502 02:04:09.147157 15216 tablet_copy_service.cc:433] P 26a3d5ced97d4dcc919b80fde53f6a5b: ending tablet copy session f77c2a969bde45b49ac1bbbe242ff171-5804597cc507483dba622e6ec44e5ead on tablet 5804597cc507483dba622e6ec44e5ead with peer f77c2a969bde45b49ac1bbbe242ff171
I20240502 02:04:09.161506 19108 consensus_queue.cc:579] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Leader has been unable to successfully communicate with peer d4e5422f66454511ae5a50ec6ab134ad for more than 300 seconds (300.489s)
W20240502 02:04:09.162724 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20240502 02:04:09.163120 19127 ts_tablet_manager.cc:902] T fc07e430e3e74e3885f263881bd04976 P 84987ad9e4a54d96af8569e4aaf0ed64: Initiating tablet copy from peer 7b66a8940fe54e21bec1b2db25b2fed5 (127.7.221.68:34323)
F20240502 02:04:09.163338 19127 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-091d27d51acc4db0a1c7d20deeff5c10.queue_time_us, tablet-download-pool-091d27d51acc4db0a1c7d20deeff5c10.queue_time_us) (tablet-download-pool-091d27d51acc4db0a1c7d20deeff5c10.run_cpu_time_us, tablet-download-pool-091d27d51acc4db0a1c7d20deeff5c10.run_cpu_time_us) (tablet-download-pool-091d27d51acc4db0a1c7d20deeff5c10.run_wall_time_us, tablet-download-pool-091d27d51acc4db0a1c7d20deeff5c10.run_wall_time_us) (tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.queue_time_us, tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.queue_time_us) (tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.run_cpu_time_us, tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.run_cpu_time_us) (tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.run_wall_time_us, tablet-download-pool-3f938c5af51a43c9a23b753c4991e41d.run_wall_time_us) (tablet-download-pool-6c4e85e7050b4dc8a4ce6bba818854a2.queue_time_us, tablet-download-pool-6c4e85e7050b4dc8a4ce6bba818854a2.queue_time_us) (tablet-download-pool-6c4e85e7050b4dc8a4ce6bba818854a2.run_cpu_time_us, tablet-download-pool-6c4e85e7050b4dc8a4ce6bba818854a2.run_cpu_time_us) (tablet-download-pool-6c4e85e7050b4dc8a4ce6bba818854a2.run_wall_time_us, tablet-download-pool-6c4e85e7050b4dc8a4ce6bba818854a2.run_wall_time_us) (tablet-download-pool-6d5cffb7fcc54118861463e3b1fa4a8d.queue_time_us, tablet-download-pool-6d5cffb7fcc54118861463e3b1fa4a8d.queue_time_us) (tablet-download-pool-6d5cffb7fcc54118861463e3b1fa4a8d.run_cpu_time_us, tablet-download-pool-6d5cffb7fcc54118861463e3b1fa4a8d.run_cpu_time_us) (tablet-download-pool-6d5cffb7fcc54118861463e3b1fa4a8d.run_wall_time_us, tablet-download-pool-6d5cffb7fcc54118861463e3b1fa4a8d.run_wall_time_us) (tablet-download-pool-77ae087802144c45924c1b79f561afe8.queue_time_us, tablet-download-pool-77ae087802144c45924c1b79f561afe8.queue_time_us) (tablet-download-pool-77ae087802144c45924c1b79f561afe8.run_cpu_time_us, tablet-download-pool-77ae087802144c45924c1b79f561afe8.run_cpu_time_us) (tablet-download-pool-77ae087802144c45924c1b79f561afe8.run_wall_time_us, tablet-download-pool-77ae087802144c45924c1b79f561afe8.run_wall_time_us) (tablet-download-pool-856f623374824d94913db0cb9faa25d1.queue_time_us, tablet-download-pool-856f623374824d94913db0cb9faa25d1.queue_time_us) (tablet-download-pool-856f623374824d94913db0cb9faa25d1.run_cpu_time_us, tablet-download-pool-856f623374824d94913db0cb9faa25d1.run_cpu_time_us) (tablet-download-pool-856f623374824d94913db0cb9faa25d1.run_wall_time_us, tablet-download-pool-856f623374824d94913db0cb9faa25d1.run_wall_time_us) (tablet-download-pool-8ddc239a297a46e48e069b985cb7b350.queue_time_us, tablet-download-pool-8ddc239a297a46e48e069b985cb7b350.queue_time_us) (tablet-download-pool-8ddc239a297a46e48e069b985cb7b350.run_cpu_time_us, tablet-download-pool-8ddc239a297a46e48e069b985cb7b350.run_cpu_time_us) (tablet-download-pool-8ddc239a297a46e48e069b985cb7b350.run_wall_time_us, tablet-download-pool-8ddc239a297a46e48e069b985cb7b350.run_wall_time_us) (tablet-download-pool-9c2892024ce942e1bf1834c1b9ad2049.queue_time_us, tablet-download-pool-9c2892024ce942e1bf1834c1b9ad2049.queue_time_us) (tablet-download-pool-9c2892024ce942e1bf1834c1b9ad2049.run_cpu_time_us, tablet-download-pool-9c2892024ce942e1bf1834c1b9ad2049.run_cpu_time_us) (tablet-download-pool-9c2892024ce942e1bf1834c1b9ad2049.run_wall_time_us, tablet-download-pool-9c2892024ce942e1bf1834c1b9ad2049.run_wall_time_us) (tablet-download-pool-d1c832c721fb4b15bc6d266a1c295401.queue_time_us, tablet-download-pool-d1c832c721fb4b15bc6d266a1c295401.queue_time_us) (tablet-download-pool-d1c832c721fb4b15bc6d266a1c295401.run_cpu_time_us, tablet-download-pool-d1c832c721fb4b15bc6d266a1c295401.run_cpu_time_us) (tablet-download-pool-d1c832c721fb4b15bc6d266a1c295401.run_wall_time_us, tablet-download-pool-d1c832c721fb4b15bc6d266a1c295401.run_wall_time_us) (tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.queue_time_us, tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.queue_time_us) (tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.run_cpu_time_us, tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.run_cpu_time_us) (tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.run_wall_time_us, tablet-download-pool-d24986c2ebab46f086e3f1f5a53af26b.run_wall_time_us) (tablet-download-pool-e00d1ed5cde044cb9bdebf787590261a.queue_time_us, tablet-download-pool-e00d1ed5cde044cb9bdebf787590261a.queue_time_us) (tablet-download-pool-e00d1ed5cde044cb9bdebf787590261a.run_cpu_time_us, tablet-download-pool-e00d1ed5cde044cb9bdebf787590261a.run_cpu_time_us) (tablet-download-pool-e00d1ed5cde044cb9bdebf787590261a.run_wall_time_us, tablet-download-pool-e00d1ed5cde044cb9bdebf787590261a.run_wall_time_us) (tablet-download-pool-e2f4ca025890492880608c7975650969.queue_time_us, tablet-download-pool-e2f4ca025890492880608c7975650969.queue_time_us) (tablet-download-pool-e2f4ca025890492880608c7975650969.run_cpu_time_us, tablet-download-pool-e2f4ca025890492880608c7975650969.run_cpu_time_us) (tablet-download-pool-e2f4ca025890492880608c7975650969.run_wall_time_us, tablet-download-pool-e2f4ca025890492880608c7975650969.run_wall_time_us) (tablet-download-pool-f97c5c9bb4c84957857efc76d1c7717f.queue_time_us, tablet-download-pool-f97c5c9bb4c84957857efc76d1c7717f.queue_time_us) (tablet-download-pool-f97c5c9bb4c84957857efc76d1c7717f.run_cpu_time_us, tablet-download-pool-f97c5c9bb4c84957857efc76d1c7717f.run_cpu_time_us) (tablet-download-pool-f97c5c9bb4c84957857efc76d1c7717f.run_wall_time_us, tablet-download-pool-f97c5c9bb4c84957857efc76d1c7717f.run_wall_time_us) (tablet-download-pool-fc07e430e3e74e3885f263881bd04976.queue_time_us, tablet-download-pool-fc07e430e3e74e3885f263881bd04976.queue_time_us) (tablet-download-pool-fcfc8dc4b90a4a6aa7e4035bbfdb4532.queue_time_us, tablet-download-pool-fcfc8dc4b90a4a6aa7e4035bbfdb4532.queue_time_us) (tablet-download-pool-fcfc8dc4b90a4a6aa7e4035bbfdb4532.run_cpu_time_us, tablet-download-pool-fcfc8dc4b90a4a6aa7e4035bbfdb4532.run_cpu_time_us) (tablet-download-pool-fcfc8dc4b90a4a6aa7e4035bbfdb4532.run_wall_time_us, tablet-download-pool-fcfc8dc4b90a4a6aa7e4035bbfdb4532.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
I20240502 02:04:09.164036 15057 consensus_queue.cc:237] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } }
I20240502 02:04:09.165854 15194 raft_consensus.cc:1261] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Refusing update from remote peer ddae2a636423489693c12b7d7d166c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
W20240502 02:04:09.166211 14995 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:09.166357 19108 consensus_queue.cc:1035] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
@ 0x7f2d6b3d0dcd google::LogMessage::Fail() at ??:0
W20240502 02:04:09.166854 14994 consensus_peers.cc:485] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:09.168437 19108 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
@ 0x7f2d6b3d4b93 google::LogMessage::SendToLog() at ??:0
I20240502 02:04:09.169936 14901 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet d1c832c721fb4b15bc6d266a1c295401 with cas_config_opid_index 10: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
@ 0x7f2d6b3d07cc google::LogMessage::Flush() at ??:0
I20240502 02:04:09.170850 14915 catalog_manager.cc:5484] T d1c832c721fb4b15bc6d266a1c295401 P ddae2a636423489693c12b7d7d166c09 reported cstate change: config changed from index 10 to 11, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "ddae2a636423489693c12b7d7d166c09" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:09.170423 15194 raft_consensus.cc:2932] T d1c832c721fb4b15bc6d266a1c295401 P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "84987ad9e4a54d96af8569e4aaf0ed64" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 35223 } attrs { promote: true } } }
@ 0x7f2d6b3d1f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f2d6c83a044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7f2d6c81ef98 kudu::ThreadPool::ThreadPool() at ??:0
I20240502 02:04:09.180399 15329 raft_consensus.cc:1203] T 8f2fb504a49c44edac92fd935555c278 P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.8->[1.9-1.9] Dedup: 1.9->[]
@ 0x7f2d6c81cefc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7f2d6f3cf77a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7f2d6f3d8a45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
I20240502 02:04:09.190474 19004 raft_consensus.cc:1013] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [term 1 LEADER]: attempt to promote peer 7b66a8940fe54e21bec1b2db25b2fed5: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
@ 0x7f2d6f45976f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7f2d6f46d4f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7f2d6f458226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7f2d6f467c0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x564b8504875c std::function<>::operator()() at ??:0
@ 0x7f2d6c821e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f2d6c8226ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f2d6c82403b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x564b8504875c std::function<>::operator()() at ??:0
@ 0x7f2d6c811807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7f2d6d04f6db start_thread at ??:0
@ 0x7f2d6a58771f clone at ??:0
I20240502 02:04:09.227636 15378 maintenance_manager.cc:419] P f77c2a969bde45b49ac1bbbe242ff171: Scheduling UndoDeltaBlockGCOp(5804597cc507483dba622e6ec44e5ead): 4834 bytes on disk
I20240502 02:04:09.228133 15267 maintenance_manager.cc:643] P f77c2a969bde45b49ac1bbbe242ff171: UndoDeltaBlockGCOp(5804597cc507483dba622e6ec44e5ead) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":93,"lbm_reads_lt_1ms":4}
I20240502 02:04:09.233898 18957 consensus_queue.cc:579] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b [LEADER]: Leader has been unable to successfully communicate with peer d4e5422f66454511ae5a50ec6ab134ad for more than 300 seconds (300.277s)
W20240502 02:04:09.235859 15130 consensus_peers.cc:485] T a901eb3063ea4911a822552db01d365d P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer d4e5422f66454511ae5a50ec6ab134ad (127.7.221.70:34649): Couldn't send request to peer d4e5422f66454511ae5a50ec6ab134ad. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.70:34649: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20240502 02:04:09.247828 14902 catalog_manager.cc:4573] TS 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet e00d1ed5cde044cb9bdebf787590261a: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111)
W20240502 02:04:09.281178 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 02:04:09.296617 15399 consensus_peers.cc:485] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
I20240502 02:04:09.358852 19116 consensus_queue.cc:579] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Leader has been unable to successfully communicate with peer 3b8d615c71e44ff887e0f037c3efcff7 for more than 300 seconds (300.080s)
W20240502 02:04:09.360247 15266 consensus_peers.cc:485] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 3b8d615c71e44ff887e0f037c3efcff7 (127.7.221.72:37645): Couldn't send request to peer 3b8d615c71e44ff887e0f037c3efcff7. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.72:37645: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20240502 02:04:09.381080 19182 ts_tablet_manager.cc:902] T 30620f4774b8414eb4e27bd5ecb6ecee P ddae2a636423489693c12b7d7d166c09: Initiating tablet copy from peer f77c2a969bde45b49ac1bbbe242ff171 (127.7.221.67:44157)
I20240502 02:04:09.381701 19182 tablet_copy_client.cc:308] T 30620f4774b8414eb4e27bd5ecb6ecee P ddae2a636423489693c12b7d7d166c09: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:44157
I20240502 02:04:09.382196 15351 tablet_copy_service.cc:140] P f77c2a969bde45b49ac1bbbe242ff171: Received BeginTabletCopySession request for tablet 30620f4774b8414eb4e27bd5ecb6ecee from peer ddae2a636423489693c12b7d7d166c09 ({username='slave'} at 127.7.221.65:46773)
I20240502 02:04:09.382316 15351 tablet_copy_service.cc:161] P f77c2a969bde45b49ac1bbbe242ff171: Beginning new tablet copy session on tablet 30620f4774b8414eb4e27bd5ecb6ecee from peer ddae2a636423489693c12b7d7d166c09 at {username='slave'} at 127.7.221.65:46773: session id = ddae2a636423489693c12b7d7d166c09-30620f4774b8414eb4e27bd5ecb6ecee
I20240502 02:04:09.383638 15351 tablet_copy_source_session.cc:205] T 30620f4774b8414eb4e27bd5ecb6ecee P f77c2a969bde45b49ac1bbbe242ff171: Tablet Copy: opened 5 blocks and 1 log segments
I20240502 02:04:09.384742 19182 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 30620f4774b8414eb4e27bd5ecb6ecee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:09.386629 19182 tablet_copy_client.cc:783] T 30620f4774b8414eb4e27bd5ecb6ecee P ddae2a636423489693c12b7d7d166c09: tablet copy: Starting download of 5 data blocks...
W20240502 02:04:09.390689 15399 connection.cc:535] client connection to 127.7.221.69:35223 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 02:04:09.390767 14995 connection.cc:535] client connection to 127.7.221.69:35223 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 02:04:09.391605 14995 consensus_peers.cc:485] T 6c4e85e7050b4dc8a4ce6bba818854a2 P ddae2a636423489693c12b7d7d166c09 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.69:35223: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 02:04:09.391607 15399 consensus_peers.cc:462] T fc07e430e3e74e3885f263881bd04976 P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.7.221.69:35223: connect: Connection refused (error 111)
W20240502 02:04:09.391680 15399 consensus_peers.cc:485] T 6d5cffb7fcc54118861463e3b1fa4a8d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.69:35223: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:09.392064 19116 consensus_queue.cc:579] T ce85785e670a4264b44797bb27cb4ecb P f77c2a969bde45b49ac1bbbe242ff171 [LEADER]: Leader has been unable to successfully communicate with peer d4e5422f66454511ae5a50ec6ab134ad for more than 300 seconds (300.114s)
I20240502 02:04:09.392076 19182 tablet_copy_client.cc:647] T 30620f4774b8414eb4e27bd5ecb6ecee P ddae2a636423489693c12b7d7d166c09: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:09.393872 19182 tablet_copy_client.cc:523] T 30620f4774b8414eb4e27bd5ecb6ecee P ddae2a636423489693c12b7d7d166c09: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:09.395507 19182 tablet_bootstrap.cc:492] T 30620f4774b8414eb4e27bd5ecb6ecee P ddae2a636423489693c12b7d7d166c09: Bootstrap starting.
I20240502 02:04:09.406423 19182 tablet_bootstrap.cc:492] T 30620f4774b8414eb4e27bd5ecb6ecee P ddae2a636423489693c12b7d7d166c09: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=2} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:09.406783 19182 tablet_bootstrap.cc:492] T 30620f4774b8414eb4e27bd5ecb6ecee P ddae2a636423489693c12b7d7d166c09: Bootstrap complete.
I20240502 02:04:09.406982 19182 ts_tablet_manager.cc:1367] T 30620f4774b8414eb4e27bd5ecb6ecee P ddae2a636423489693c12b7d7d166c09: Time spent bootstrapping tablet: real 0.012s user 0.008s sys 0.003s
I20240502 02:04:09.407424 19182 raft_consensus.cc:348] T 30620f4774b8414eb4e27bd5ecb6ecee P ddae2a636423489693c12b7d7d166c09 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } }
I20240502 02:04:09.407554 19182 raft_consensus.cc:729] T 30620f4774b8414eb4e27bd5ecb6ecee P ddae2a636423489693c12b7d7d166c09 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: ddae2a636423489693c12b7d7d166c09, State: Initialized, Role: LEARNER
I20240502 02:04:09.407719 19182 consensus_queue.cc:260] T 30620f4774b8414eb4e27bd5ecb6ecee P ddae2a636423489693c12b7d7d166c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f77c2a969bde45b49ac1bbbe242ff171" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 44157 } } peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "d4e5422f66454511ae5a50ec6ab134ad" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 34649 } attrs { promote: false } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 46123 } attrs { promote: true } }
I20240502 02:04:09.408056 19182 ts_tablet_manager.cc:1398] T 30620f4774b8414eb4e27bd5ecb6ecee P ddae2a636423489693c12b7d7d166c09: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:09.408584 15351 tablet_copy_service.cc:342] P f77c2a969bde45b49ac1bbbe242ff171: Request end of tablet copy session ddae2a636423489693c12b7d7d166c09-30620f4774b8414eb4e27bd5ecb6ecee received from {username='slave'} at 127.7.221.65:46773
I20240502 02:04:09.408658 15351 tablet_copy_service.cc:433] P f77c2a969bde45b49ac1bbbe242ff171: ending tablet copy session ddae2a636423489693c12b7d7d166c09-30620f4774b8414eb4e27bd5ecb6ecee on tablet 30620f4774b8414eb4e27bd5ecb6ecee with peer ddae2a636423489693c12b7d7d166c09
I20240502 02:04:09.411592 15110 maintenance_manager.cc:419] P ddae2a636423489693c12b7d7d166c09: Scheduling UndoDeltaBlockGCOp(30620f4774b8414eb4e27bd5ecb6ecee): 4834 bytes on disk
I20240502 02:04:09.411980 14996 maintenance_manager.cc:643] P ddae2a636423489693c12b7d7d166c09: UndoDeltaBlockGCOp(30620f4774b8414eb4e27bd5ecb6ecee) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
W20240502 02:04:09.443106 15131 consensus_peers.cc:485] T f97c5c9bb4c84957857efc76d1c7717f P 26a3d5ced97d4dcc919b80fde53f6a5b -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.69:35223: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:09.469795 15174 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 02:04:09.470165 15444 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 02:04:09.475486 15309 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
W20240502 02:04:09.478905 15266 consensus_peers.cc:485] T 9c2892024ce942e1bf1834c1b9ad2049 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.69:35223: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:09.482892 15040 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
W20240502 02:04:09.500418 14995 consensus_peers.cc:485] T 77ae087802144c45924c1b79f561afe8 P ddae2a636423489693c12b7d7d166c09 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.69:35223: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 02:04:09.511847 15266 consensus_peers.cc:485] T 091d27d51acc4db0a1c7d20deeff5c10 P f77c2a969bde45b49ac1bbbe242ff171 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.69:35223: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:09.515200 17299 rebalancer_tool-test.cc:1260] rebalancer run info: Runtime error: /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu: process exited with non-zero status 1
stdout: Per-server replica distribution summary:
Statistic | Value
-----------------------+-----------
Minimum Replica Count | 0
Maximum Replica Count | 23
Average Replica Count | 11.250000
Per-table replica distribution summary:
Replica Skew | Value
--------------+----------
Minimum | 3
Maximum | 3
Average | 3.000000
stderr: W20240502 01:59:03.168296 17302 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 01:59:03.169732 17302 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 01:59:03.183092 17302 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 01:59:04.139910 17302 rebalancer_tool.cc:215] running whole-cluster rebalancing
W20240502 01:59:04.695618 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:04.695735 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.695992 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.696085 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.696198 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.696286 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.696359 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 12ee6dc62f814bd7b3c13bee4495d9e0: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.696424 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 9736e7c5365849759564e386d208932f: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.696516 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0b29d58fe9c74d1cb4345bb6cfbd0535: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.696588 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:04.696681 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:04.696749 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:04.696820 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:04.696892 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:04.735357 17302 rebalancer_tool.cc:1084] tablet fcfc8dc4b90a4a6aa7e4035bbfdb4532: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'd4e5422f66454511ae5a50ec6ab134ad' move ignored: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } }.
Pending config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "26a3d5ced97d4dcc919b80fde53f6a5b" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34905 } attrs { replace: true } } peers { permanent_uuid: "ddae2a636423489693c12b7d7d166c09" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 46123 } } peers { permanent_uuid: "7b66a8940fe54e21bec1b2db25b2fed5" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 34323 } } peers { permanent_uuid: "fc067ee690bb457e8109ad8e4eb43ab8" member_type: NON_VOTER last_known_addr { host: "127.7.221.71" port: 43207 } attrs { promote: true } }
I20240502 01:59:04.735457 17302 rebalancer_tool.cc:756] re-synchronizing cluster state
W20240502 01:59:05.065448 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:05.065521 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:05.065557 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:05.065627 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.065663 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:05.065697 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.065742 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0b29d58fe9c74d1cb4345bb6cfbd0535: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:05.065810 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 2739aac0519d438587e8ec91530f5579: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.065860 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1c35c23ce11f4c249032c42d4c499e42: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:05.065896 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 3bc69a0a1e474b9fac7330d68770bfbe: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server fc067ee690bb457e8109ad8e4eb43ab8
W20240502 01:59:05.065929 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 73bf3d4779b34c7290b42f7f561e8e11: could not find any suitable replica to move from server 7b66a8940fe54e21bec1b2db25b2fed5 to server d4e5422f66454511ae5a50ec6ab134ad
W20240502 01:59:05.065963 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table bf7d5082701149bea6b9e5c480f53c10: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:05.066015 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server 26a3d5ced97d4dcc919b80fde53f6a5b to server 84987ad9e4a54d96af8569e4aaf0ed64
I20240502 01:59:05.095759 17302 rebalancer_tool.cc:1074] tablet a901eb3063ea4911a822552db01d365d: 'f77c2a969bde45b49ac1bbbe242ff171' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:05.176400 17302 rebalancer_tool.cc:1074] tablet e2f4ca025890492880608c7975650969: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:05.196244 17302 rebalancer_tool.cc:1074] tablet 6d5cffb7fcc54118861463e3b1fa4a8d: 'ddae2a636423489693c12b7d7d166c09' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move scheduled
I20240502 01:59:05.233475 17302 rebalancer_tool.cc:1074] tablet e00d1ed5cde044cb9bdebf787590261a: 'f77c2a969bde45b49ac1bbbe242ff171' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:05.261250 17302 rebalancer_tool.cc:1074] tablet 81c8fe21ec7545e79450532b43a61ee7: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:05.402575 17302 rebalancer_tool.cc:1074] tablet 1eb15147ff6c4d0b83b12dec736fc576: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:05.502586 17302 rebalancer_tool.cc:1074] tablet d24986c2ebab46f086e3f1f5a53af26b: 'ddae2a636423489693c12b7d7d166c09' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move scheduled
I20240502 01:59:05.563153 17302 rebalancer_tool.cc:1074] tablet d1c832c721fb4b15bc6d266a1c295401: 'f77c2a969bde45b49ac1bbbe242ff171' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:05.782474 17302 rebalancer_tool.cc:1074] tablet 986e7c97e0734e958f364dc5d1167846: '7b66a8940fe54e21bec1b2db25b2fed5' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:05.993701 17302 rebalancer_tool.cc:1074] tablet 3f938c5af51a43c9a23b753c4991e41d: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:06.117290 17302 rebalancer_tool.cc:1074] tablet ce85785e670a4264b44797bb27cb4ecb: 'ddae2a636423489693c12b7d7d166c09' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move scheduled
I20240502 01:59:06.253998 17302 rebalancer_tool.cc:1074] tablet 53edbd87d0d5471ca7626ca2071f75eb: 'f77c2a969bde45b49ac1bbbe242ff171' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:06.318499 17302 rebalancer_tool.cc:1084] tablet 30620f4774b8414eb4e27bd5ecb6ecee: '7b66a8940fe54e21bec1b2db25b2fed5' -> '3b8d615c71e44ff887e0f037c3efcff7' move ignored: Illegal state: tablet 30620f4774b8414eb4e27bd5ecb6ecee: replica 3b8d615c71e44ff887e0f037c3efcff7 is already present
I20240502 01:59:06.318583 17302 rebalancer_tool.cc:756] re-synchronizing cluster state
W20240502 01:59:07.718479 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server ddae2a636423489693c12b7d7d166c09 to server 3b8d615c71e44ff887e0f037c3efcff7
W20240502 01:59:07.718607 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server fc067ee690bb457e8109ad8e4eb43ab8 to server 84987ad9e4a54d96af8569e4aaf0ed64
W20240502 01:59:07.718736 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 1fafbde99fe04ac28cefeef0570a2828: could not find any suitable replica to move from server f77c2a969bde45b49ac1bbbe242ff171 to server fc067ee690bb457e8109ad8e4eb43ab8
I20240502 01:59:07.740969 17302 rebalancer_tool.cc:1074] tablet 856f623374824d94913db0cb9faa25d1: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '' move scheduled
I20240502 01:59:07.768152 17302 rebalancer_tool.cc:1074] tablet 8ddc239a297a46e48e069b985cb7b350: 'ddae2a636423489693c12b7d7d166c09' -> '' move scheduled
I20240502 01:59:07.901265 17302 rebalancer_tool.cc:1074] tablet 5804597cc507483dba622e6ec44e5ead: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:07.969695 17302 rebalancer_tool.cc:1074] tablet 8f2fb504a49c44edac92fd935555c278: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:08.088138 17302 rebalancer_tool.cc:1074] tablet 490e37fac0844a08a07a4b281bc33bf9: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move scheduled
I20240502 01:59:08.205128 17302 rebalancer_tool.cc:1084] tablet 21e2e55d1a0d4b6eb18b4ade9468607f: 'f77c2a969bde45b49ac1bbbe242ff171' -> 'd4e5422f66454511ae5a50ec6ab134ad' move ignored: Illegal state: tablet 21e2e55d1a0d4b6eb18b4ade9468607f: replica d4e5422f66454511ae5a50ec6ab134ad is already present
I20240502 01:59:08.205210 17302 rebalancer_tool.cc:756] re-synchronizing cluster state
W20240502 01:59:08.731655 17302 rebalancer_tool.cc:1205] No replica could be moved this iteration: Not found: table 0fd3b28283df457db390a2882ebcfef5: could not find any suitable replica to move from server 3b8d615c71e44ff887e0f037c3efcff7 to server d4e5422f66454511ae5a50ec6ab134ad
I20240502 01:59:08.963490 17302 rebalancer_tool.cc:1074] tablet 6c4e85e7050b4dc8a4ce6bba818854a2: '84987ad9e4a54d96af8569e4aaf0ed64' -> '' move scheduled
I20240502 01:59:09.142735 17302 rebalancer_tool.cc:1074] tablet 21e2e55d1a0d4b6eb18b4ade9468607f: 'd4e5422f66454511ae5a50ec6ab134ad' -> 'ddae2a636423489693c12b7d7d166c09' move scheduled
I20240502 01:59:09.207312 17302 rebalancer_tool.cc:1123] tablet 1eb15147ff6c4d0b83b12dec736fc576: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '3b8d615c71e44ff887e0f037c3efcff7' move is abandoned: Illegal state: 1eb15147ff6c4d0b83b12dec736fc576: source replica 26a3d5ced97d4dcc919b80fde53f6a5b does not have REPLACE attribute set
I20240502 01:59:09.219938 17302 rebalancer_tool.cc:1136] tablet 3f938c5af51a43c9a23b753c4991e41d: '26a3d5ced97d4dcc919b80fde53f6a5b' -> 'd4e5422f66454511ae5a50ec6ab134ad' move completed: OK
I20240502 01:59:09.241952 17302 rebalancer_tool.cc:1136] tablet 81c8fe21ec7545e79450532b43a61ee7: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'd4e5422f66454511ae5a50ec6ab134ad' move completed: OK
I20240502 01:59:09.271759 17302 rebalancer_tool.cc:1136] tablet e00d1ed5cde044cb9bdebf787590261a: 'f77c2a969bde45b49ac1bbbe242ff171' -> '84987ad9e4a54d96af8569e4aaf0ed64' move completed: OK
I20240502 01:59:09.282478 17302 rebalancer_tool.cc:1136] tablet 6d5cffb7fcc54118861463e3b1fa4a8d: 'ddae2a636423489693c12b7d7d166c09' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move completed: OK
I20240502 01:59:09.290161 17302 rebalancer_tool.cc:1136] tablet e2f4ca025890492880608c7975650969: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '3b8d615c71e44ff887e0f037c3efcff7' move completed: Incomplete: tablet e2f4ca025890492880608c7975650969, TS 26a3d5ced97d4dcc919b80fde53f6a5b -> TS 3b8d615c71e44ff887e0f037c3efcff7 move failed, target replica disappeared
I20240502 01:59:09.380041 17302 rebalancer_tool.cc:1136] tablet 53edbd87d0d5471ca7626ca2071f75eb: 'f77c2a969bde45b49ac1bbbe242ff171' -> '84987ad9e4a54d96af8569e4aaf0ed64' move completed: Incomplete: tablet 53edbd87d0d5471ca7626ca2071f75eb, TS f77c2a969bde45b49ac1bbbe242ff171 -> TS 84987ad9e4a54d96af8569e4aaf0ed64 move failed, target replica disappeared
I20240502 01:59:09.403576 17302 rebalancer_tool.cc:1136] tablet a901eb3063ea4911a822552db01d365d: 'f77c2a969bde45b49ac1bbbe242ff171' -> 'd4e5422f66454511ae5a50ec6ab134ad' move completed: OK
I20240502 01:59:09.480450 17302 rebalancer_tool.cc:1136] tablet d24986c2ebab46f086e3f1f5a53af26b: 'ddae2a636423489693c12b7d7d166c09' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move completed: OK
I20240502 01:59:09.519827 17302 rebalancer_tool.cc:1136] tablet d1c832c721fb4b15bc6d266a1c295401: 'f77c2a969bde45b49ac1bbbe242ff171' -> '84987ad9e4a54d96af8569e4aaf0ed64' move completed: Incomplete: tablet d1c832c721fb4b15bc6d266a1c295401, TS f77c2a969bde45b49ac1bbbe242ff171 -> TS 84987ad9e4a54d96af8569e4aaf0ed64 move failed, target replica disappeared
I20240502 01:59:09.545348 17302 rebalancer_tool.cc:1136] tablet ce85785e670a4264b44797bb27cb4ecb: 'ddae2a636423489693c12b7d7d166c09' -> 'fc067ee690bb457e8109ad8e4eb43ab8' move completed: OK
I20240502 01:59:09.593793 17302 rebalancer_tool.cc:1074] tablet 9c2892024ce942e1bf1834c1b9ad2049: '84987ad9e4a54d96af8569e4aaf0ed64' -> 'd4e5422f66454511ae5a50ec6ab134ad' move scheduled
I20240502 01:59:09.609567 17302 rebalancer_tool.cc:1074] tablet 091d27d51acc4db0a1c7d20deeff5c10: 'd4e5422f66454511ae5a50ec6ab134ad' -> '84987ad9e4a54d96af8569e4aaf0ed64' move scheduled
I20240502 01:59:09.638231 17302 rebalancer_tool.cc:1074] tablet 6db29f2516e1486782be0312c9bcac8b: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '7b66a8940fe54e21bec1b2db25b2fed5' move scheduled
I20240502 01:59:09.663434 17302 rebalancer_tool.cc:1074] tablet 77ae087802144c45924c1b79f561afe8: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '3b8d615c71e44ff887e0f037c3efcff7' move scheduled
I20240502 01:59:10.864118 17302 rebalancer_tool.cc:1136] tablet 21e2e55d1a0d4b6eb18b4ade9468607f: 'd4e5422f66454511ae5a50ec6ab134ad' -> 'ddae2a636423489693c12b7d7d166c09' move completed: OK
I20240502 01:59:11.159970 17302 rebalancer_tool.cc:1136] tablet 091d27d51acc4db0a1c7d20deeff5c10: 'd4e5422f66454511ae5a50ec6ab134ad' -> '84987ad9e4a54d96af8569e4aaf0ed64' move completed: OK
I20240502 02:04:08.431572 17302 rebalancer_tool.cc:1136] tablet 8f2fb504a49c44edac92fd935555c278: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '3b8d615c71e44ff887e0f037c3efcff7' move completed: Incomplete: tablet 8f2fb504a49c44edac92fd935555c278, TS 26a3d5ced97d4dcc919b80fde53f6a5b -> TS 3b8d615c71e44ff887e0f037c3efcff7 move failed, target replica disappeared
I20240502 02:04:08.719743 17302 rebalancer_tool.cc:1136] tablet 5804597cc507483dba622e6ec44e5ead: '7b66a8940fe54e21bec1b2db25b2fed5' -> 'd4e5422f66454511ae5a50ec6ab134ad' move completed: Incomplete: tablet 5804597cc507483dba622e6ec44e5ead, TS 7b66a8940fe54e21bec1b2db25b2fed5 -> TS d4e5422f66454511ae5a50ec6ab134ad move failed, target replica disappeared
W20240502 02:04:09.392401 19173 negotiation.cc:337] Failed RPC negotiation. Trace:
0502 02:04:09.244636 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.7.221.69:35223 (local address 127.0.0.1:35242)
0502 02:04:09.245152 (+ 516us) negotiation.cc:107] Waiting for socket to connect
0502 02:04:09.245164 (+ 12us) client_negotiation.cc:174] Beginning negotiation
0502 02:04:09.245231 (+ 67us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
0502 02:04:09.391334 (+146103us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.7.221.69:35223: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
Metrics: {"client-negotiator.queue_time_us":140,"thread_start_us":71,"threads_started":1}
I20240502 02:04:09.392804 17302 rebalancer_tool.cc:1123] tablet 856f623374824d94913db0cb9faa25d1: '26a3d5ced97d4dcc919b80fde53f6a5b' -> '' move is abandoned: Network error: Client connection negotiation failed: client connection to 127.7.221.69:35223: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 02:04:09.395604 17302 rebalancer_tool.cc:1123] tablet 8ddc239a297a46e48e069b985cb7b350: 'ddae2a636423489693c12b7d7d166c09' -> '' move is abandoned: Network error: Client connection negotiation failed: client connection to 127.7.221.69:35223: connect: Connection refused (error 111)
I20240502 02:04:09.401701 17302 rebalancer_tool.cc:756] re-synchronizing cluster state
Illegal state: tablet server 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): unacceptable health status UNAVAILABLE
E20240502 02:04:09.515683 8053 external_mini_cluster.cc:943] Process with UUID 84987ad9e4a54d96af8569e4aaf0ed64 has crashed
E20240502 02:04:09.515764 8053 external_mini_cluster.cc:943] Process with UUID d4e5422f66454511ae5a50ec6ab134ad has crashed
E20240502 02:04:09.515811 8053 external_mini_cluster.cc:943] Process with UUID fc067ee690bb457e8109ad8e4eb43ab8 has crashed
E20240502 02:04:09.515861 8053 external_mini_cluster.cc:943] Process with UUID 3b8d615c71e44ff887e0f037c3efcff7 has crashed
/home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/mini-cluster/external_mini_cluster.cc:947: Failure
Expected equality of these values:
0
num_crashes
Which is: 4
At least one process crashed
/home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/tools/rebalancer_tool-test.cc:1284: Failure
Expected: cluster_->AssertNoCrashes() doesn't generate new fatal failures in the current thread.
Actual: it does.
I20240502 02:04:09.516538 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 14977
I20240502 02:04:09.518422 15329 raft_consensus.cc:1203] T 5804597cc507483dba622e6ec44e5ead P f77c2a969bde45b49ac1bbbe242ff171 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240502 02:04:09.526294 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 15112
W20240502 02:04:09.529079 15399 consensus_peers.cc:485] T 3f938c5af51a43c9a23b753c4991e41d P 7b66a8940fe54e21bec1b2db25b2fed5 -> Peer 84987ad9e4a54d96af8569e4aaf0ed64 (127.7.221.69:35223): Couldn't send request to peer 84987ad9e4a54d96af8569e4aaf0ed64. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.69:35223: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:09.535166 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 15246
I20240502 02:04:09.536844 18961 consensus_queue.cc:579] T ac5ec2c673cb4b988d1099e0442ec63f P 7b66a8940fe54e21bec1b2db25b2fed5 [LEADER]: Leader has been unable to successfully communicate with peer fc067ee690bb457e8109ad8e4eb43ab8 for more than 300 seconds (300.368s)
I20240502 02:04:09.543532 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 15380
I20240502 02:04:09.551730 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 14883
2024-05-02T02:04:09Z chronyd exiting
I20240502 02:04:09.569140 8053 test_util.cc:175] -----------------------------------------------
I20240502 02:04:09.569185 8053 test_util.cc:176] Had fatal failures, leaving test files at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.ConcurrentRebalancersTest.TwoConcurrentRebalancers_1.1714615092046232-8053-0
[ FAILED ] ConcurrentRebalancersTest.TwoConcurrentRebalancers/1, where GetParam() = 4-byte object <01-00 00-00> (317901 ms)
[----------] 1 test from ConcurrentRebalancersTest (317901 ms total)
[----------] 1 test from RebalancerAndSingleReplicaTablets
[ RUN ] RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove/1
I20240502 02:04:09.569785 8053 test_util.cc:268] Using random seed: 80497143
I20240502 02:04:09.570466 8053 ts_itest-base.cc:112] Starting cluster with:
I20240502 02:04:09.570500 8053 ts_itest-base.cc:113] --------------
I20240502 02:04:09.570511 8053 ts_itest-base.cc:114] 4 tablet servers
I20240502 02:04:09.570521 8053 ts_itest-base.cc:115] 1 replicas per TS
I20240502 02:04:09.570530 8053 ts_itest-base.cc:116] --------------
2024-05-02T02:04:09Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T02:04:09Z Disabled control of system clock
I20240502 02:04:09.578318 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.7.221.126:43599
--webserver_interface=127.7.221.126
--webserver_port=0
--builtin_ntp_servers=127.7.221.84:36725
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.7.221.126:43599
--raft_prepare_replacement_before_eviction=true
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false with env {}
W20240502 02:04:09.679744 19244 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:09.679956 19244 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:09.680035 19244 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:09.682799 19244 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:09.682868 19244 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 02:04:09.682912 19244 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:09.682947 19244 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 02:04:09.682981 19244 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 02:04:09.686393 19244 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36725
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.7.221.126:43599
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.7.221.126:43599
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.7.221.126
--webserver_port=0
--never_fsync=true
--heap_profile_path=/tmp/kudu.19244
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:09.687150 19244 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:09.688163 19244 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:09.695230 19250 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:09.695827 19252 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:09.696018 19249 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:09.696714 19244 server_base.cc:1034] running on GCE node
I20240502 02:04:09.697096 19244 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:09.697834 19244 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:09.698973 19244 hybrid_clock.cc:648] HybridClock initialized: now 1714615449698958 us; error 26 us; skew 500 ppm
I20240502 02:04:09.700584 19244 webserver.cc:460] Webserver started at http://127.7.221.126:41515/ using document root <none> and password file <none>
I20240502 02:04:09.701057 19244 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:09.701105 19244 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:09.701278 19244 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:09.702807 19244 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "47aad5d269304635b35d456d7e868c6b"
format_stamp: "Formatted at 2024-05-02 02:04:09 on dist-test-slave-dmxt"
I20240502 02:04:09.703208 19244 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "47aad5d269304635b35d456d7e868c6b"
format_stamp: "Formatted at 2024-05-02 02:04:09 on dist-test-slave-dmxt"
I20240502 02:04:09.706156 19244 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 02:04:09.708041 19259 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:09.708889 19244 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.001s
I20240502 02:04:09.708981 19244 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
uuid: "47aad5d269304635b35d456d7e868c6b"
format_stamp: "Formatted at 2024-05-02 02:04:09 on dist-test-slave-dmxt"
I20240502 02:04:09.709064 19244 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:09.720896 19244 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:09.721375 19244 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:09.721513 19244 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:09.727504 19244 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.126:43599
I20240502 02:04:09.727514 19311 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.126:43599 every 8 connection(s)
I20240502 02:04:09.728279 19244 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 02:04:09.730479 19312 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:09.732967 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 19244
I20240502 02:04:09.733093 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 02:04:09.735502 19312 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b: Bootstrap starting.
I20240502 02:04:09.737332 19312 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:09.738093 19312 log.cc:824] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:09.739698 19312 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b: No bootstrap required, opened a new log
I20240502 02:04:09.742286 19312 raft_consensus.cc:348] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "47aad5d269304635b35d456d7e868c6b" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43599 } }
I20240502 02:04:09.742429 19312 raft_consensus.cc:374] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:09.742472 19312 raft_consensus.cc:729] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 47aad5d269304635b35d456d7e868c6b, State: Initialized, Role: FOLLOWER
I20240502 02:04:09.742863 19312 consensus_queue.cc:260] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "47aad5d269304635b35d456d7e868c6b" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43599 } }
I20240502 02:04:09.742980 19312 raft_consensus.cc:388] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:09.743037 19312 raft_consensus.cc:482] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:09.743146 19312 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:09.743783 19312 raft_consensus.cc:504] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "47aad5d269304635b35d456d7e868c6b" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43599 } }
I20240502 02:04:09.744073 19312 leader_election.cc:302] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 47aad5d269304635b35d456d7e868c6b; no voters:
I20240502 02:04:09.744341 19312 leader_election.cc:288] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:09.744663 19317 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:09.744848 19317 raft_consensus.cc:686] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [term 1 LEADER]: Becoming Leader. State: Replica: 47aad5d269304635b35d456d7e868c6b, State: Running, Role: LEADER
I20240502 02:04:09.745087 19317 consensus_queue.cc:237] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "47aad5d269304635b35d456d7e868c6b" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43599 } }
I20240502 02:04:09.745325 19312 sys_catalog.cc:564] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [sys.catalog]: configured and running, proceeding with master startup.
I20240502 02:04:09.746359 19318 sys_catalog.cc:455] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "47aad5d269304635b35d456d7e868c6b" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "47aad5d269304635b35d456d7e868c6b" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43599 } } }
I20240502 02:04:09.746492 19318 sys_catalog.cc:458] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [sys.catalog]: This master's current role is: LEADER
I20240502 02:04:09.746773 19319 sys_catalog.cc:455] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 47aad5d269304635b35d456d7e868c6b. Latest consensus state: current_term: 1 leader_uuid: "47aad5d269304635b35d456d7e868c6b" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "47aad5d269304635b35d456d7e868c6b" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43599 } } }
I20240502 02:04:09.746840 19327 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 02:04:09.746857 19319 sys_catalog.cc:458] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b [sys.catalog]: This master's current role is: LEADER
I20240502 02:04:09.749536 19327 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 02:04:09.753795 19327 catalog_manager.cc:1347] Generated new cluster ID: 1406bf090cb94f5396264fb47c4078d3
I20240502 02:04:09.753852 19327 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 02:04:09.766911 19327 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 02:04:09.767627 19327 catalog_manager.cc:1504] Loading token signing keys...
I20240502 02:04:09.771811 19327 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 47aad5d269304635b35d456d7e868c6b: Generated new TSK 0
I20240502 02:04:09.772503 19327 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 02:04:09.782455 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.65:0
--local_ip_for_outbound_sockets=127.7.221.65
--webserver_interface=127.7.221.65
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43599
--builtin_ntp_servers=127.7.221.84:36725
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 02:04:09.880844 19337 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:09.881026 19337 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:09.881094 19337 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:09.883883 19337 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:09.883944 19337 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:09.884034 19337 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.65
I20240502 02:04:09.887275 19337 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36725
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.7.221.65
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43599
--never_fsync=true
--heap_profile_path=/tmp/kudu.19337
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.65
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:09.888047 19337 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:09.889034 19337 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:09.896327 19342 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:09.896903 19345 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:09.897611 19343 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:09.898164 19337 server_base.cc:1034] running on GCE node
I20240502 02:04:09.898470 19337 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:09.898958 19337 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:09.900099 19337 hybrid_clock.cc:648] HybridClock initialized: now 1714615449900081 us; error 38 us; skew 500 ppm
I20240502 02:04:09.901604 19337 webserver.cc:460] Webserver started at http://127.7.221.65:41927/ using document root <none> and password file <none>
I20240502 02:04:09.902102 19337 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:09.902159 19337 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:09.902310 19337 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:09.903765 19337 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "e29339b99789441ebc37b6f327ad34ae"
format_stamp: "Formatted at 2024-05-02 02:04:09 on dist-test-slave-dmxt"
I20240502 02:04:09.904175 19337 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "e29339b99789441ebc37b6f327ad34ae"
format_stamp: "Formatted at 2024-05-02 02:04:09 on dist-test-slave-dmxt"
I20240502 02:04:09.907078 19337 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.001s sys 0.002s
I20240502 02:04:09.908964 19351 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:09.909819 19337 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:09.909922 19337 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "e29339b99789441ebc37b6f327ad34ae"
format_stamp: "Formatted at 2024-05-02 02:04:09 on dist-test-slave-dmxt"
I20240502 02:04:09.909993 19337 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:09.924485 19337 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:09.924968 19337 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:09.925110 19337 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:09.925558 19337 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:09.926281 19337 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:09.926324 19337 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:09.926371 19337 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:09.926399 19337 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:09.933969 19337 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.65:36593
I20240502 02:04:09.933992 19466 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.65:36593 every 8 connection(s)
I20240502 02:04:09.934768 19337 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 02:04:09.937973 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 19337
I20240502 02:04:09.938081 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 02:04:09.940109 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.66:0
--local_ip_for_outbound_sockets=127.7.221.66
--webserver_interface=127.7.221.66
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43599
--builtin_ntp_servers=127.7.221.84:36725
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 02:04:09.942936 19467 heartbeater.cc:347] Connected to a master server at 127.7.221.126:43599
I20240502 02:04:09.943100 19467 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:09.943519 19467 heartbeater.cc:511] Master 127.7.221.126:43599 requested a full tablet report, sending...
I20240502 02:04:09.944808 19276 ts_manager.cc:194] Registered new tserver with Master: e29339b99789441ebc37b6f327ad34ae (127.7.221.65:36593)
I20240502 02:04:09.945946 19276 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.65:38203
W20240502 02:04:10.038874 19470 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:10.039077 19470 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:10.039158 19470 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:10.041937 19470 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:10.041996 19470 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:10.042079 19470 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.66
I20240502 02:04:10.045511 19470 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36725
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.7.221.66
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43599
--never_fsync=true
--heap_profile_path=/tmp/kudu.19470
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.66
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:10.046306 19470 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:10.047385 19470 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:10.055215 19475 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:10.055235 19478 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:10.055871 19476 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:10.056535 19470 server_base.cc:1034] running on GCE node
I20240502 02:04:10.056862 19470 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:10.057322 19470 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:10.058457 19470 hybrid_clock.cc:648] HybridClock initialized: now 1714615450058430 us; error 41 us; skew 500 ppm
I20240502 02:04:10.060228 19470 webserver.cc:460] Webserver started at http://127.7.221.66:38713/ using document root <none> and password file <none>
I20240502 02:04:10.060705 19470 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:10.060760 19470 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:10.060911 19470 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:10.062350 19470 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "d2faf854fabf42afa2c5355009726e51"
format_stamp: "Formatted at 2024-05-02 02:04:10 on dist-test-slave-dmxt"
I20240502 02:04:10.062834 19470 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "d2faf854fabf42afa2c5355009726e51"
format_stamp: "Formatted at 2024-05-02 02:04:10 on dist-test-slave-dmxt"
I20240502 02:04:10.065891 19470 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.001s sys 0.001s
I20240502 02:04:10.067732 19484 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:10.068563 19470 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.001s
I20240502 02:04:10.068666 19470 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "d2faf854fabf42afa2c5355009726e51"
format_stamp: "Formatted at 2024-05-02 02:04:10 on dist-test-slave-dmxt"
I20240502 02:04:10.068737 19470 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:10.086773 19470 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:10.087230 19470 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:10.087361 19470 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:10.087801 19470 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:10.088589 19470 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:10.088639 19470 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:10.088689 19470 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:10.088711 19470 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:10.096511 19470 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.66:39613
I20240502 02:04:10.096540 19601 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.66:39613 every 8 connection(s)
I20240502 02:04:10.097283 19470 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 02:04:10.104938 19602 heartbeater.cc:347] Connected to a master server at 127.7.221.126:43599
I20240502 02:04:10.105129 19602 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:10.105574 19602 heartbeater.cc:511] Master 127.7.221.126:43599 requested a full tablet report, sending...
I20240502 02:04:10.105659 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 19470
I20240502 02:04:10.105803 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 02:04:10.106551 19276 ts_manager.cc:194] Registered new tserver with Master: d2faf854fabf42afa2c5355009726e51 (127.7.221.66:39613)
I20240502 02:04:10.107226 19276 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.66:35549
I20240502 02:04:10.107970 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.67:0
--local_ip_for_outbound_sockets=127.7.221.67
--webserver_interface=127.7.221.67
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43599
--builtin_ntp_servers=127.7.221.84:36725
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 02:04:10.207707 19605 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:10.207895 19605 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:10.207933 19605 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:10.210619 19605 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:10.210693 19605 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:10.210765 19605 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.67
I20240502 02:04:10.214076 19605 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36725
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.7.221.67
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43599
--never_fsync=true
--heap_profile_path=/tmp/kudu.19605
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.67
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:10.214921 19605 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:10.215963 19605 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:10.223214 19610 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:10.223779 19614 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:10.224503 19611 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:10.225167 19605 server_base.cc:1034] running on GCE node
I20240502 02:04:10.225483 19605 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:10.225970 19605 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:10.227106 19605 hybrid_clock.cc:648] HybridClock initialized: now 1714615450227096 us; error 24 us; skew 500 ppm
I20240502 02:04:10.228818 19605 webserver.cc:460] Webserver started at http://127.7.221.67:34203/ using document root <none> and password file <none>
I20240502 02:04:10.229306 19605 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:10.229363 19605 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:10.229519 19605 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:10.231065 19605 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15"
format_stamp: "Formatted at 2024-05-02 02:04:10 on dist-test-slave-dmxt"
I20240502 02:04:10.231551 19605 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15"
format_stamp: "Formatted at 2024-05-02 02:04:10 on dist-test-slave-dmxt"
I20240502 02:04:10.234762 19605 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 02:04:10.236804 19620 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:10.237663 19605 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:10.237759 19605 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15"
format_stamp: "Formatted at 2024-05-02 02:04:10 on dist-test-slave-dmxt"
I20240502 02:04:10.237833 19605 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:10.249279 19605 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:10.249816 19605 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:10.249979 19605 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:10.250427 19605 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:10.251264 19605 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:10.251319 19605 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:10.251374 19605 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:10.251400 19605 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:10.259312 19605 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.67:33495
I20240502 02:04:10.259351 19736 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.67:33495 every 8 connection(s)
I20240502 02:04:10.260143 19605 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 02:04:10.263187 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 19605
I20240502 02:04:10.263304 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 02:04:10.265756 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.68:0
--local_ip_for_outbound_sockets=127.7.221.68
--webserver_interface=127.7.221.68
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43599
--builtin_ntp_servers=127.7.221.84:36725
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 02:04:10.272786 19737 heartbeater.cc:347] Connected to a master server at 127.7.221.126:43599
I20240502 02:04:10.273025 19737 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:10.273682 19737 heartbeater.cc:511] Master 127.7.221.126:43599 requested a full tablet report, sending...
I20240502 02:04:10.274682 19276 ts_manager.cc:194] Registered new tserver with Master: 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67:33495)
I20240502 02:04:10.275221 19276 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.67:55639
W20240502 02:04:10.365523 19740 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:10.365739 19740 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:10.365777 19740 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:10.368494 19740 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:10.368547 19740 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:10.368602 19740 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.68
I20240502 02:04:10.371776 19740 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36725
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.68:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.7.221.68
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43599
--never_fsync=true
--heap_profile_path=/tmp/kudu.19740
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.68
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:10.372556 19740 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:10.373530 19740 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:10.380856 19745 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:10.381387 19746 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:10.382100 19748 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:10.382761 19740 server_base.cc:1034] running on GCE node
I20240502 02:04:10.383105 19740 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:10.383630 19740 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:10.384783 19740 hybrid_clock.cc:648] HybridClock initialized: now 1714615450384762 us; error 42 us; skew 500 ppm
I20240502 02:04:10.386579 19740 webserver.cc:460] Webserver started at http://127.7.221.68:46481/ using document root <none> and password file <none>
I20240502 02:04:10.387147 19740 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:10.387207 19740 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:10.387377 19740 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:10.388842 19740 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec"
format_stamp: "Formatted at 2024-05-02 02:04:10 on dist-test-slave-dmxt"
I20240502 02:04:10.389299 19740 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec"
format_stamp: "Formatted at 2024-05-02 02:04:10 on dist-test-slave-dmxt"
I20240502 02:04:10.392481 19740 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.000s sys 0.005s
I20240502 02:04:10.394572 19754 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:10.395550 19740 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.003s
I20240502 02:04:10.395653 19740 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec"
format_stamp: "Formatted at 2024-05-02 02:04:10 on dist-test-slave-dmxt"
I20240502 02:04:10.395746 19740 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:10.412959 19740 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:10.413527 19740 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:10.413671 19740 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:10.414127 19740 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:10.415051 19740 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:10.415103 19740 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:10.415164 19740 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:10.415191 19740 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:10.422773 19740 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.68:36343
I20240502 02:04:10.422885 19868 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.68:36343 every 8 connection(s)
I20240502 02:04:10.423637 19740 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 02:04:10.431366 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 19740
I20240502 02:04:10.431360 19869 heartbeater.cc:347] Connected to a master server at 127.7.221.126:43599
I20240502 02:04:10.431488 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 02:04:10.431582 19869 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:10.432152 19869 heartbeater.cc:511] Master 127.7.221.126:43599 requested a full tablet report, sending...
I20240502 02:04:10.433241 19275 ts_manager.cc:194] Registered new tserver with Master: 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68:36343)
I20240502 02:04:10.433848 19275 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.68:51235
I20240502 02:04:10.445178 8053 external_mini_cluster.cc:930] 4 TS(s) registered with all masters
I20240502 02:04:10.453152 19275 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51620:
name: "TestTable"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
owner: "alice"
I20240502 02:04:10.467713 19532 tablet_service.cc:1467] Processing CreateTablet for tablet f788ee8d1d2043c8b7860111eafffba8 (DEFAULT_TABLE table=TestTable [id=b4ae69ef11e8451d9d1ffd54708f6f87]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:10.468516 19532 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f788ee8d1d2043c8b7860111eafffba8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:10.473235 19882 tablet_bootstrap.cc:492] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51: Bootstrap starting.
I20240502 02:04:10.475271 19882 tablet_bootstrap.cc:654] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:10.475936 19882 log.cc:824] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:10.477303 19882 tablet_bootstrap.cc:492] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51: No bootstrap required, opened a new log
I20240502 02:04:10.477463 19882 ts_tablet_manager.cc:1367] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 02:04:10.480235 19882 raft_consensus.cc:348] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:10.480399 19882 raft_consensus.cc:374] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:10.480468 19882 raft_consensus.cc:729] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Initialized, Role: FOLLOWER
I20240502 02:04:10.480841 19882 consensus_queue.cc:260] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:10.480962 19882 raft_consensus.cc:388] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:10.481019 19882 raft_consensus.cc:482] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:10.481134 19882 raft_consensus.cc:3037] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:10.481851 19882 raft_consensus.cc:504] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:10.482146 19882 leader_election.cc:302] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: d2faf854fabf42afa2c5355009726e51; no voters:
I20240502 02:04:10.482411 19882 leader_election.cc:288] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:10.482503 19884 raft_consensus.cc:2781] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:10.482772 19884 raft_consensus.cc:686] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Becoming Leader. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Running, Role: LEADER
I20240502 02:04:10.483063 19884 consensus_queue.cc:237] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:10.483114 19602 heartbeater.cc:503] Master 127.7.221.126:43599 was elected leader, sending a full tablet report...
I20240502 02:04:10.483249 19882 ts_tablet_manager.cc:1398] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51: Time spent starting tablet: real 0.006s user 0.006s sys 0.000s
I20240502 02:04:10.486138 19275 catalog_manager.cc:5484] T f788ee8d1d2043c8b7860111eafffba8 P d2faf854fabf42afa2c5355009726e51 reported cstate change: term changed from 0 to 1, leader changed from <none> to d2faf854fabf42afa2c5355009726e51 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "d2faf854fabf42afa2c5355009726e51" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:10.493397 8053 external_mini_cluster.cc:930] 4 TS(s) registered with all masters
I20240502 02:04:10.494683 8053 ts_itest-base.cc:233] Waiting for 0 tablets on tserver e29339b99789441ebc37b6f327ad34ae to finish bootstrapping
I20240502 02:04:10.500473 8053 ts_itest-base.cc:233] Waiting for 1 tablets on tserver d2faf854fabf42afa2c5355009726e51 to finish bootstrapping
I20240502 02:04:10.504942 8053 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 5ba05ad7c35a41f4b1a0cb4bef0d6d15 to finish bootstrapping
I20240502 02:04:10.510195 8053 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 6ddcf3d8a67f4601b88c7cdab5fc6fec to finish bootstrapping
I20240502 02:04:10.515291 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 19605
I20240502 02:04:10.521589 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 19740
I20240502 02:04:10.947786 19467 heartbeater.cc:503] Master 127.7.221.126:43599 was elected leader, sending a full tablet report...
I20240502 02:04:14.279183 19275 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51620:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 02:04:14.290392 19397 tablet_service.cc:1467] Processing CreateTablet for tablet bc4cb3dc8c3340a88fbc50e175c68f49 (DEFAULT_TABLE table=rebalance_test_table_0 [id=55a2f39752324cc5843da57dfc176da0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:14.290387 19398 tablet_service.cc:1467] Processing CreateTablet for tablet cdae5597120c4631b6c2ffa7176d60c9 (DEFAULT_TABLE table=rebalance_test_table_0 [id=55a2f39752324cc5843da57dfc176da0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:14.290392 19399 tablet_service.cc:1467] Processing CreateTablet for tablet 15436057adfe47679cbe78552d8c29b0 (DEFAULT_TABLE table=rebalance_test_table_0 [id=55a2f39752324cc5843da57dfc176da0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:14.291476 19399 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 15436057adfe47679cbe78552d8c29b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:14.291793 19398 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cdae5597120c4631b6c2ffa7176d60c9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:14.291986 19397 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bc4cb3dc8c3340a88fbc50e175c68f49. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:14.296636 19900 tablet_bootstrap.cc:492] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae: Bootstrap starting.
I20240502 02:04:14.298868 19900 tablet_bootstrap.cc:654] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:14.299530 19900 log.cc:824] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:14.300981 19900 tablet_bootstrap.cc:492] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae: No bootstrap required, opened a new log
I20240502 02:04:14.301162 19900 ts_tablet_manager.cc:1367] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240502 02:04:14.303596 19900 raft_consensus.cc:348] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.303768 19900 raft_consensus.cc:374] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:14.303817 19900 raft_consensus.cc:729] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Initialized, Role: FOLLOWER
I20240502 02:04:14.304172 19900 consensus_queue.cc:260] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.304311 19900 raft_consensus.cc:388] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:14.304373 19900 raft_consensus.cc:482] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:14.304481 19900 raft_consensus.cc:3037] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:14.305178 19900 raft_consensus.cc:504] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.305465 19900 leader_election.cc:302] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e29339b99789441ebc37b6f327ad34ae; no voters:
I20240502 02:04:14.305722 19900 leader_election.cc:288] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:14.305814 19902 raft_consensus.cc:2781] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:14.306017 19902 raft_consensus.cc:686] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Becoming Leader. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Running, Role: LEADER
I20240502 02:04:14.306296 19902 consensus_queue.cc:237] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.306444 19900 ts_tablet_manager.cc:1398] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae: Time spent starting tablet: real 0.005s user 0.006s sys 0.000s
I20240502 02:04:14.306680 19900 tablet_bootstrap.cc:492] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae: Bootstrap starting.
I20240502 02:04:14.307643 19900 tablet_bootstrap.cc:654] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:14.308737 19900 tablet_bootstrap.cc:492] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae: No bootstrap required, opened a new log
I20240502 02:04:14.308832 19900 ts_tablet_manager.cc:1367] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:14.309345 19900 raft_consensus.cc:348] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.309417 19900 raft_consensus.cc:374] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:14.309438 19900 raft_consensus.cc:729] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Initialized, Role: FOLLOWER
I20240502 02:04:14.309352 19275 catalog_manager.cc:5484] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae reported cstate change: term changed from 0 to 1, leader changed from <none> to e29339b99789441ebc37b6f327ad34ae (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:14.309530 19900 consensus_queue.cc:260] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.309595 19900 raft_consensus.cc:388] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:14.309623 19900 raft_consensus.cc:482] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:14.309669 19900 raft_consensus.cc:3037] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:14.310235 19900 raft_consensus.cc:504] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.310348 19900 leader_election.cc:302] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e29339b99789441ebc37b6f327ad34ae; no voters:
I20240502 02:04:14.310441 19900 leader_election.cc:288] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:14.310483 19902 raft_consensus.cc:2781] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:14.310581 19902 raft_consensus.cc:686] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Becoming Leader. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Running, Role: LEADER
I20240502 02:04:14.310739 19900 ts_tablet_manager.cc:1398] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:14.310720 19902 consensus_queue.cc:237] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.310887 19900 tablet_bootstrap.cc:492] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae: Bootstrap starting.
I20240502 02:04:14.311700 19900 tablet_bootstrap.cc:654] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:14.312800 19275 catalog_manager.cc:5484] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae reported cstate change: term changed from 0 to 1, leader changed from <none> to e29339b99789441ebc37b6f327ad34ae (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:14.313072 19900 tablet_bootstrap.cc:492] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae: No bootstrap required, opened a new log
I20240502 02:04:14.313167 19900 ts_tablet_manager.cc:1367] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:14.313576 19900 raft_consensus.cc:348] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.313675 19900 raft_consensus.cc:374] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:14.313710 19900 raft_consensus.cc:729] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Initialized, Role: FOLLOWER
I20240502 02:04:14.313833 19900 consensus_queue.cc:260] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.313920 19900 raft_consensus.cc:388] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:14.313959 19900 raft_consensus.cc:482] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:14.314008 19900 raft_consensus.cc:3037] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:14.314834 19900 raft_consensus.cc:504] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.314967 19900 leader_election.cc:302] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e29339b99789441ebc37b6f327ad34ae; no voters:
I20240502 02:04:14.315063 19900 leader_election.cc:288] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:14.315133 19902 raft_consensus.cc:2781] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:14.315227 19902 raft_consensus.cc:686] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Becoming Leader. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Running, Role: LEADER
I20240502 02:04:14.315265 19900 ts_tablet_manager.cc:1398] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:14.315333 19902 consensus_queue.cc:237] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.316365 19275 catalog_manager.cc:5484] T 15436057adfe47679cbe78552d8c29b0 P e29339b99789441ebc37b6f327ad34ae reported cstate change: term changed from 0 to 1, leader changed from <none> to e29339b99789441ebc37b6f327ad34ae (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:14.482419 19275 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51620:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 02:04:14.487816 19531 tablet_service.cc:1467] Processing CreateTablet for tablet d77eb55afc0e4dd796390ebb3961d0fc (DEFAULT_TABLE table=rebalance_test_table_1 [id=2653c0a691044c738cd2eea4ca609aa3]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:14.487809 19532 tablet_service.cc:1467] Processing CreateTablet for tablet 118dc310317a4c6b8fdbeb0a710e62b0 (DEFAULT_TABLE table=rebalance_test_table_1 [id=2653c0a691044c738cd2eea4ca609aa3]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:14.487804 19530 tablet_service.cc:1467] Processing CreateTablet for tablet c07aa9911ae24eb7b8221e5245ad56be (DEFAULT_TABLE table=rebalance_test_table_1 [id=2653c0a691044c738cd2eea4ca609aa3]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:14.488219 19531 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d77eb55afc0e4dd796390ebb3961d0fc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:14.488380 19530 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c07aa9911ae24eb7b8221e5245ad56be. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:14.489899 19532 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 118dc310317a4c6b8fdbeb0a710e62b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:14.490182 19930 tablet_bootstrap.cc:492] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51: Bootstrap starting.
I20240502 02:04:14.491046 19930 tablet_bootstrap.cc:654] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:14.492172 19930 tablet_bootstrap.cc:492] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51: No bootstrap required, opened a new log
I20240502 02:04:14.492237 19930 ts_tablet_manager.cc:1367] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 02:04:14.492637 19930 raft_consensus.cc:348] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.492715 19930 raft_consensus.cc:374] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:14.492740 19930 raft_consensus.cc:729] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Initialized, Role: FOLLOWER
I20240502 02:04:14.492839 19930 consensus_queue.cc:260] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.492921 19930 raft_consensus.cc:388] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:14.492966 19930 raft_consensus.cc:482] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:14.493011 19930 raft_consensus.cc:3037] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:14.493706 19930 raft_consensus.cc:504] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.493808 19930 leader_election.cc:302] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: d2faf854fabf42afa2c5355009726e51; no voters:
I20240502 02:04:14.493916 19930 leader_election.cc:288] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:14.494027 19932 raft_consensus.cc:2781] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:14.494140 19930 ts_tablet_manager.cc:1398] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 02:04:14.494211 19932 raft_consensus.cc:686] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Becoming Leader. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Running, Role: LEADER
I20240502 02:04:14.494266 19930 tablet_bootstrap.cc:492] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51: Bootstrap starting.
I20240502 02:04:14.494340 19932 consensus_queue.cc:237] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.495074 19930 tablet_bootstrap.cc:654] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:14.495674 19275 catalog_manager.cc:5484] T c07aa9911ae24eb7b8221e5245ad56be P d2faf854fabf42afa2c5355009726e51 reported cstate change: term changed from 0 to 1, leader changed from <none> to d2faf854fabf42afa2c5355009726e51 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "d2faf854fabf42afa2c5355009726e51" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:14.496026 19930 tablet_bootstrap.cc:492] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51: No bootstrap required, opened a new log
I20240502 02:04:14.496116 19930 ts_tablet_manager.cc:1367] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 02:04:14.496490 19930 raft_consensus.cc:348] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.496579 19930 raft_consensus.cc:374] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:14.496634 19930 raft_consensus.cc:729] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Initialized, Role: FOLLOWER
I20240502 02:04:14.496740 19930 consensus_queue.cc:260] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.496821 19930 raft_consensus.cc:388] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:14.496851 19930 raft_consensus.cc:482] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:14.496888 19930 raft_consensus.cc:3037] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:14.497740 19930 raft_consensus.cc:504] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.497856 19930 leader_election.cc:302] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: d2faf854fabf42afa2c5355009726e51; no voters:
I20240502 02:04:14.497954 19930 leader_election.cc:288] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:14.497997 19932 raft_consensus.cc:2781] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:14.498088 19932 raft_consensus.cc:686] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Becoming Leader. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Running, Role: LEADER
I20240502 02:04:14.498160 19930 ts_tablet_manager.cc:1398] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 02:04:14.498217 19932 consensus_queue.cc:237] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.498329 19930 tablet_bootstrap.cc:492] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51: Bootstrap starting.
I20240502 02:04:14.499119 19930 tablet_bootstrap.cc:654] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:14.499779 19275 catalog_manager.cc:5484] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 reported cstate change: term changed from 0 to 1, leader changed from <none> to d2faf854fabf42afa2c5355009726e51 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "d2faf854fabf42afa2c5355009726e51" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:14.500061 19930 tablet_bootstrap.cc:492] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51: No bootstrap required, opened a new log
I20240502 02:04:14.500330 19930 ts_tablet_manager.cc:1367] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:14.500686 19930 raft_consensus.cc:348] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.500773 19930 raft_consensus.cc:374] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:14.500811 19930 raft_consensus.cc:729] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Initialized, Role: FOLLOWER
I20240502 02:04:14.500941 19930 consensus_queue.cc:260] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.501020 19930 raft_consensus.cc:388] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:14.501042 19930 raft_consensus.cc:482] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:14.501085 19930 raft_consensus.cc:3037] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:14.501775 19930 raft_consensus.cc:504] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.501870 19930 leader_election.cc:302] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: d2faf854fabf42afa2c5355009726e51; no voters:
I20240502 02:04:14.501955 19930 leader_election.cc:288] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:14.502004 19933 raft_consensus.cc:2781] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:14.502081 19933 raft_consensus.cc:686] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Becoming Leader. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Running, Role: LEADER
I20240502 02:04:14.502151 19930 ts_tablet_manager.cc:1398] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:14.502182 19933 consensus_queue.cc:237] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.503232 19275 catalog_manager.cc:5484] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 reported cstate change: term changed from 0 to 1, leader changed from <none> to d2faf854fabf42afa2c5355009726e51 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "d2faf854fabf42afa2c5355009726e51" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:14.661456 19275 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51620:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 02:04:14.667057 19398 tablet_service.cc:1467] Processing CreateTablet for tablet d9243491bd8142339827c5b7718cdfd1 (DEFAULT_TABLE table=rebalance_test_table_2 [id=e95f8116d31f4b4b8526822b427924ee]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:14.667156 19397 tablet_service.cc:1467] Processing CreateTablet for tablet c7aa1292397343e69abf05758d4eca7f (DEFAULT_TABLE table=rebalance_test_table_2 [id=e95f8116d31f4b4b8526822b427924ee]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:14.667061 19399 tablet_service.cc:1467] Processing CreateTablet for tablet 61234e24e53049f5be6c60dbbd170328 (DEFAULT_TABLE table=rebalance_test_table_2 [id=e95f8116d31f4b4b8526822b427924ee]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:14.667452 19398 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d9243491bd8142339827c5b7718cdfd1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:14.667671 19399 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 61234e24e53049f5be6c60dbbd170328. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:14.667843 19397 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c7aa1292397343e69abf05758d4eca7f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:14.669636 19900 tablet_bootstrap.cc:492] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae: Bootstrap starting.
I20240502 02:04:14.670456 19900 tablet_bootstrap.cc:654] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:14.671602 19900 tablet_bootstrap.cc:492] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae: No bootstrap required, opened a new log
I20240502 02:04:14.671707 19900 ts_tablet_manager.cc:1367] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:14.672137 19900 raft_consensus.cc:348] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.672269 19900 raft_consensus.cc:374] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:14.672319 19900 raft_consensus.cc:729] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Initialized, Role: FOLLOWER
I20240502 02:04:14.672446 19900 consensus_queue.cc:260] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.672536 19900 raft_consensus.cc:388] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:14.672582 19900 raft_consensus.cc:482] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:14.672622 19900 raft_consensus.cc:3037] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:14.673257 19900 raft_consensus.cc:504] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.673362 19900 leader_election.cc:302] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e29339b99789441ebc37b6f327ad34ae; no voters:
I20240502 02:04:14.673432 19900 leader_election.cc:288] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:14.673503 19902 raft_consensus.cc:2781] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:14.673581 19902 raft_consensus.cc:686] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Becoming Leader. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Running, Role: LEADER
I20240502 02:04:14.673676 19900 ts_tablet_manager.cc:1398] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:14.673696 19902 consensus_queue.cc:237] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.673823 19900 tablet_bootstrap.cc:492] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae: Bootstrap starting.
I20240502 02:04:14.674741 19900 tablet_bootstrap.cc:654] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:14.675130 19275 catalog_manager.cc:5484] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae reported cstate change: term changed from 0 to 1, leader changed from <none> to e29339b99789441ebc37b6f327ad34ae (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:14.675936 19900 tablet_bootstrap.cc:492] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae: No bootstrap required, opened a new log
I20240502 02:04:14.676025 19900 ts_tablet_manager.cc:1367] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 02:04:14.676395 19900 raft_consensus.cc:348] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.676481 19900 raft_consensus.cc:374] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:14.676502 19900 raft_consensus.cc:729] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Initialized, Role: FOLLOWER
I20240502 02:04:14.676594 19900 consensus_queue.cc:260] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.676657 19900 raft_consensus.cc:388] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:14.676685 19900 raft_consensus.cc:482] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:14.676721 19900 raft_consensus.cc:3037] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:14.677433 19900 raft_consensus.cc:504] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.677549 19900 leader_election.cc:302] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e29339b99789441ebc37b6f327ad34ae; no voters:
I20240502 02:04:14.677644 19900 leader_election.cc:288] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:14.677690 19902 raft_consensus.cc:2781] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:14.677773 19902 raft_consensus.cc:686] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Becoming Leader. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Running, Role: LEADER
I20240502 02:04:14.677874 19902 consensus_queue.cc:237] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.677994 19900 ts_tablet_manager.cc:1398] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 02:04:14.678141 19900 tablet_bootstrap.cc:492] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae: Bootstrap starting.
I20240502 02:04:14.679106 19900 tablet_bootstrap.cc:654] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:14.679441 19275 catalog_manager.cc:5484] T d9243491bd8142339827c5b7718cdfd1 P e29339b99789441ebc37b6f327ad34ae reported cstate change: term changed from 0 to 1, leader changed from <none> to e29339b99789441ebc37b6f327ad34ae (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:14.680094 19900 tablet_bootstrap.cc:492] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae: No bootstrap required, opened a new log
I20240502 02:04:14.680171 19900 ts_tablet_manager.cc:1367] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:14.680536 19900 raft_consensus.cc:348] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.680605 19900 raft_consensus.cc:374] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:14.680629 19900 raft_consensus.cc:729] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Initialized, Role: FOLLOWER
I20240502 02:04:14.680756 19900 consensus_queue.cc:260] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.680840 19900 raft_consensus.cc:388] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:14.680867 19900 raft_consensus.cc:482] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:14.680920 19900 raft_consensus.cc:3037] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:14.681674 19900 raft_consensus.cc:504] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.681785 19900 leader_election.cc:302] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e29339b99789441ebc37b6f327ad34ae; no voters:
I20240502 02:04:14.681879 19900 leader_election.cc:288] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:14.681910 19902 raft_consensus.cc:2781] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:14.681984 19902 raft_consensus.cc:686] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Becoming Leader. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Running, Role: LEADER
I20240502 02:04:14.682088 19902 consensus_queue.cc:237] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } }
I20240502 02:04:14.682096 19900 ts_tablet_manager.cc:1398] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:14.683270 19275 catalog_manager.cc:5484] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae reported cstate change: term changed from 0 to 1, leader changed from <none> to e29339b99789441ebc37b6f327ad34ae (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:14.838361 19275 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51620:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 02:04:14.843289 19531 tablet_service.cc:1467] Processing CreateTablet for tablet 8bcff5fdda3c49c992c2cb78014f4673 (DEFAULT_TABLE table=rebalance_test_table_3 [id=fc293509ebb744bf9e6d16de7f985b15]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:14.843304 19530 tablet_service.cc:1467] Processing CreateTablet for tablet 021ef17cfdbc4537b9b3fbb49b240af8 (DEFAULT_TABLE table=rebalance_test_table_3 [id=fc293509ebb744bf9e6d16de7f985b15]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:14.843681 19531 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8bcff5fdda3c49c992c2cb78014f4673. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:14.843870 19530 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 021ef17cfdbc4537b9b3fbb49b240af8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:14.843286 19532 tablet_service.cc:1467] Processing CreateTablet for tablet dc40ed8ee597423c9fc8993863b83d9e (DEFAULT_TABLE table=rebalance_test_table_3 [id=fc293509ebb744bf9e6d16de7f985b15]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:14.844094 19532 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dc40ed8ee597423c9fc8993863b83d9e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:14.846199 19930 tablet_bootstrap.cc:492] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51: Bootstrap starting.
I20240502 02:04:14.847134 19930 tablet_bootstrap.cc:654] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:14.848129 19930 tablet_bootstrap.cc:492] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51: No bootstrap required, opened a new log
I20240502 02:04:14.848198 19930 ts_tablet_manager.cc:1367] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 02:04:14.848548 19930 raft_consensus.cc:348] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.848645 19930 raft_consensus.cc:374] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:14.848681 19930 raft_consensus.cc:729] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Initialized, Role: FOLLOWER
I20240502 02:04:14.848803 19930 consensus_queue.cc:260] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.848891 19930 raft_consensus.cc:388] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:14.848937 19930 raft_consensus.cc:482] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:14.848985 19930 raft_consensus.cc:3037] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:14.849659 19930 raft_consensus.cc:504] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.849786 19930 leader_election.cc:302] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: d2faf854fabf42afa2c5355009726e51; no voters:
I20240502 02:04:14.849864 19930 leader_election.cc:288] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:14.850100 19930 ts_tablet_manager.cc:1398] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 02:04:14.850128 19934 raft_consensus.cc:2781] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:14.850221 19934 raft_consensus.cc:686] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Becoming Leader. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Running, Role: LEADER
I20240502 02:04:14.850275 19930 tablet_bootstrap.cc:492] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51: Bootstrap starting.
I20240502 02:04:14.850332 19934 consensus_queue.cc:237] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.851089 19930 tablet_bootstrap.cc:654] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:14.851491 19275 catalog_manager.cc:5484] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 reported cstate change: term changed from 0 to 1, leader changed from <none> to d2faf854fabf42afa2c5355009726e51 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "d2faf854fabf42afa2c5355009726e51" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:14.851975 19930 tablet_bootstrap.cc:492] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51: No bootstrap required, opened a new log
I20240502 02:04:14.852042 19930 ts_tablet_manager.cc:1367] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:14.852377 19930 raft_consensus.cc:348] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.852465 19930 raft_consensus.cc:374] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:14.852509 19930 raft_consensus.cc:729] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Initialized, Role: FOLLOWER
I20240502 02:04:14.852638 19930 consensus_queue.cc:260] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.852716 19930 raft_consensus.cc:388] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:14.852758 19930 raft_consensus.cc:482] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:14.852813 19930 raft_consensus.cc:3037] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:14.853588 19930 raft_consensus.cc:504] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.853709 19930 leader_election.cc:302] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: d2faf854fabf42afa2c5355009726e51; no voters:
I20240502 02:04:14.853811 19930 leader_election.cc:288] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:14.853863 19934 raft_consensus.cc:2781] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:14.853950 19934 raft_consensus.cc:686] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Becoming Leader. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Running, Role: LEADER
I20240502 02:04:14.854044 19934 consensus_queue.cc:237] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.854094 19930 ts_tablet_manager.cc:1398] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:14.854249 19930 tablet_bootstrap.cc:492] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51: Bootstrap starting.
I20240502 02:04:14.855100 19930 tablet_bootstrap.cc:654] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51: Neither blocks nor log segments found. Creating new log.
W20240502 02:04:14.855269 19603 tablet.cc:2369] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 02:04:14.856101 19275 catalog_manager.cc:5484] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 reported cstate change: term changed from 0 to 1, leader changed from <none> to d2faf854fabf42afa2c5355009726e51 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "d2faf854fabf42afa2c5355009726e51" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:14.856225 19930 tablet_bootstrap.cc:492] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51: No bootstrap required, opened a new log
I20240502 02:04:14.856287 19930 ts_tablet_manager.cc:1367] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 02:04:14.856772 19930 raft_consensus.cc:348] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.856846 19930 raft_consensus.cc:374] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:14.856866 19930 raft_consensus.cc:729] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Initialized, Role: FOLLOWER
I20240502 02:04:14.856994 19930 consensus_queue.cc:260] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.857055 19930 raft_consensus.cc:388] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:14.857088 19930 raft_consensus.cc:482] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:14.857132 19930 raft_consensus.cc:3037] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:14.857774 19930 raft_consensus.cc:504] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.857903 19930 leader_election.cc:302] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: d2faf854fabf42afa2c5355009726e51; no voters:
I20240502 02:04:14.857991 19930 leader_election.cc:288] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:14.858023 19934 raft_consensus.cc:2781] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:14.858095 19934 raft_consensus.cc:686] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Becoming Leader. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Running, Role: LEADER
I20240502 02:04:14.858196 19930 ts_tablet_manager.cc:1398] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 02:04:14.858201 19934 consensus_queue.cc:237] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } }
I20240502 02:04:14.859347 19275 catalog_manager.cc:5484] T 8bcff5fdda3c49c992c2cb78014f4673 P d2faf854fabf42afa2c5355009726e51 reported cstate change: term changed from 0 to 1, leader changed from <none> to d2faf854fabf42afa2c5355009726e51 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "d2faf854fabf42afa2c5355009726e51" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:15.008956 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.67:33495
--local_ip_for_outbound_sockets=127.7.221.67
--tserver_master_addrs=127.7.221.126:43599
--webserver_port=34203
--webserver_interface=127.7.221.67
--builtin_ntp_servers=127.7.221.84:36725
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 02:04:15.111953 20007 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:15.112175 20007 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:15.112248 20007 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:15.115377 20007 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:15.115435 20007 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:15.115516 20007 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.67
I20240502 02:04:15.118958 20007 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36725
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.67:33495
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.7.221.67
--webserver_port=34203
--tserver_master_addrs=127.7.221.126:43599
--never_fsync=true
--heap_profile_path=/tmp/kudu.20007
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.67
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:15.119747 20007 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:15.120781 20007 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:15.129040 20013 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:15.129666 20015 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:15.129798 20012 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:15.130419 20007 server_base.cc:1034] running on GCE node
I20240502 02:04:15.130833 20007 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:15.131363 20007 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:15.132510 20007 hybrid_clock.cc:648] HybridClock initialized: now 1714615455132479 us; error 45 us; skew 500 ppm
I20240502 02:04:15.134326 20007 webserver.cc:460] Webserver started at http://127.7.221.67:34203/ using document root <none> and password file <none>
I20240502 02:04:15.134903 20007 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:15.134980 20007 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:15.138037 20007 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.003s sys 0.000s
I20240502 02:04:15.139838 20021 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:15.140758 20007 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:15.140861 20007 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15"
format_stamp: "Formatted at 2024-05-02 02:04:10 on dist-test-slave-dmxt"
I20240502 02:04:15.141247 20007 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:15.153210 20007 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:15.153708 20007 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:15.153861 20007 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:15.154356 20007 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:15.155221 20007 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:15.155272 20007 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:15.155326 20007 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:15.155359 20007 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:15.163280 20007 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.67:33495
I20240502 02:04:15.163308 20136 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.67:33495 every 8 connection(s)
I20240502 02:04:15.164270 20007 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 02:04:15.172765 20137 heartbeater.cc:347] Connected to a master server at 127.7.221.126:43599
I20240502 02:04:15.172979 20137 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:15.173455 20137 heartbeater.cc:511] Master 127.7.221.126:43599 requested a full tablet report, sending...
I20240502 02:04:15.174228 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 20007
I20240502 02:04:15.174502 19276 ts_manager.cc:194] Re-registered known tserver with Master: 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67:33495)
I20240502 02:04:15.174721 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.68:36343
--local_ip_for_outbound_sockets=127.7.221.68
--tserver_master_addrs=127.7.221.126:43599
--webserver_port=46481
--webserver_interface=127.7.221.68
--builtin_ntp_servers=127.7.221.84:36725
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 02:04:15.175122 19276 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.67:40295
W20240502 02:04:15.279824 20140 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:15.280038 20140 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:15.280092 20140 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:15.283144 20140 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:15.283219 20140 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:15.283341 20140 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.68
I20240502 02:04:15.286911 20140 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:36725
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.68:36343
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.7.221.68
--webserver_port=46481
--tserver_master_addrs=127.7.221.126:43599
--never_fsync=true
--heap_profile_path=/tmp/kudu.20140
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.68
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:15.287712 20140 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:15.288756 20140 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:15.296186 20145 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:15.296809 20146 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:15.297551 20148 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:15.298192 20140 server_base.cc:1034] running on GCE node
I20240502 02:04:15.298537 20140 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:15.299091 20140 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:15.300236 20140 hybrid_clock.cc:648] HybridClock initialized: now 1714615455300211 us; error 37 us; skew 500 ppm
I20240502 02:04:15.302029 20140 webserver.cc:460] Webserver started at http://127.7.221.68:46481/ using document root <none> and password file <none>
I20240502 02:04:15.302532 20140 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:15.302639 20140 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:15.305694 20140 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.000s sys 0.002s
I20240502 02:04:15.307435 20154 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:15.308358 20140 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.002s
I20240502 02:04:15.308462 20140 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec"
format_stamp: "Formatted at 2024-05-02 02:04:10 on dist-test-slave-dmxt"
I20240502 02:04:15.308830 20140 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:15.332414 20140 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:15.332968 20140 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:15.333114 20140 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:15.333596 20140 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:15.334750 20140 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:15.334822 20140 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:15.334887 20140 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:15.334918 20140 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:15.342583 20269 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.68:36343 every 8 connection(s)
I20240502 02:04:15.342617 20140 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.68:36343
I20240502 02:04:15.343623 20140 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 02:04:15.350421 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 20140
I20240502 02:04:15.351940 20270 heartbeater.cc:347] Connected to a master server at 127.7.221.126:43599
I20240502 02:04:15.352139 20270 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:15.352591 20270 heartbeater.cc:511] Master 127.7.221.126:43599 requested a full tablet report, sending...
I20240502 02:04:15.353500 19276 ts_manager.cc:194] Re-registered known tserver with Master: 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68:36343)
I20240502 02:04:15.354000 19276 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.68:35001
I20240502 02:04:15.531554 19397 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 02:04:15.531956 19531 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 02:04:15.533459 20202 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:15.534569 20070 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:15.619823 20070 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:15.620556 20202 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:15.623296 19531 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 02:04:15.623318 19397 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 02:04:15.722697 20070 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:15.724032 19531 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 02:04:15.724690 19397 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 02:04:15.724794 20202 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:15.763801 19419 consensus_queue.cc:237] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } }
I20240502 02:04:15.765194 20392 raft_consensus.cc:2932] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) added. New config: { opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } } }
I20240502 02:04:15.768289 19276 catalog_manager.cc:5484] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae reported cstate change: config changed from index -1 to 2, NON_VOTER 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) added. New cstate: current_term: 1 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 02:04:15.770112 19352 consensus_peers.cc:485] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae -> Peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67:33495): Couldn't send request to peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c7aa1292397343e69abf05758d4eca7f. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:15.776036 19555 consensus_queue.cc:237] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: true } }
I20240502 02:04:15.777339 20404 raft_consensus.cc:2932] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68) added. New config: { opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: true } } }
I20240502 02:04:15.780782 19276 catalog_manager.cc:5484] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 reported cstate change: config changed from index -1 to 2, NON_VOTER 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "d2faf854fabf42afa2c5355009726e51" committed_config { opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 02:04:15.782500 19488 consensus_peers.cc:485] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 -> Peer 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68:36343): Couldn't send request to peer 6ddcf3d8a67f4601b88c7cdab5fc6fec. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 021ef17cfdbc4537b9b3fbb49b240af8. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:15.789063 19419 consensus_queue.cc:237] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } }
I20240502 02:04:15.790014 20392 raft_consensus.cc:2932] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) added. New config: { opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } } }
W20240502 02:04:15.790418 19352 consensus_peers.cc:485] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae -> Peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67:33495): Couldn't send request to peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bc4cb3dc8c3340a88fbc50e175c68f49. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:15.792299 19276 catalog_manager.cc:5484] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae reported cstate change: config changed from index -1 to 2, NON_VOTER 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) added. New cstate: current_term: 1 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:15.799882 19555 consensus_queue.cc:237] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: true } }
I20240502 02:04:15.800829 20403 raft_consensus.cc:2932] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index -1 to 3, NON_VOTER e29339b99789441ebc37b6f327ad34ae (127.7.221.65) added. New config: { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: true } } }
I20240502 02:04:15.803135 19276 catalog_manager.cc:5484] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 reported cstate change: config changed from index -1 to 3, NON_VOTER e29339b99789441ebc37b6f327ad34ae (127.7.221.65) added. New cstate: current_term: 1 leader_uuid: "d2faf854fabf42afa2c5355009726e51" committed_config { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 02:04:15.805054 19487 consensus_peers.cc:485] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 -> Peer e29339b99789441ebc37b6f327ad34ae (127.7.221.65:36593): Couldn't send request to peer e29339b99789441ebc37b6f327ad34ae. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 118dc310317a4c6b8fdbeb0a710e62b0. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:15.810212 19419 consensus_queue.cc:237] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: true } }
I20240502 02:04:15.811262 20412 raft_consensus.cc:2932] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68) added. New config: { opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: true } } }
I20240502 02:04:15.814483 19276 catalog_manager.cc:5484] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae reported cstate change: config changed from index -1 to 2, NON_VOTER 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 02:04:15.815177 19355 consensus_peers.cc:485] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae -> Peer 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68:36343): Couldn't send request to peer 6ddcf3d8a67f4601b88c7cdab5fc6fec. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cdae5597120c4631b6c2ffa7176d60c9. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:15.821053 19555 consensus_queue.cc:237] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } }
I20240502 02:04:15.821921 20404 raft_consensus.cc:2932] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) added. New config: { opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } } }
I20240502 02:04:15.824142 19276 catalog_manager.cc:5484] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 reported cstate change: config changed from index -1 to 2, NON_VOTER 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) added. New cstate: current_term: 1 leader_uuid: "d2faf854fabf42afa2c5355009726e51" committed_config { opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 02:04:15.826110 19485 consensus_peers.cc:485] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 -> Peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67:33495): Couldn't send request to peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d77eb55afc0e4dd796390ebb3961d0fc. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:15.831281 19419 consensus_queue.cc:237] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: true } }
I20240502 02:04:15.832149 20392 raft_consensus.cc:2932] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68) added. New config: { opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: true } } }
W20240502 02:04:15.832605 19355 consensus_peers.cc:485] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae -> Peer 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68:36343): Couldn't send request to peer 6ddcf3d8a67f4601b88c7cdab5fc6fec. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 61234e24e53049f5be6c60dbbd170328. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:15.834225 19276 catalog_manager.cc:5484] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae reported cstate change: config changed from index -1 to 2, NON_VOTER 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:15.842051 19555 consensus_queue.cc:237] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } }
I20240502 02:04:15.842808 20404 raft_consensus.cc:2932] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index -1 to 3, NON_VOTER 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) added. New config: { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } } }
W20240502 02:04:15.843353 19485 consensus_peers.cc:485] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 -> Peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67:33495): Couldn't send request to peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: dc40ed8ee597423c9fc8993863b83d9e. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:15.845088 19276 catalog_manager.cc:5484] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 reported cstate change: config changed from index -1 to 3, NON_VOTER 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) added. New cstate: current_term: 1 leader_uuid: "d2faf854fabf42afa2c5355009726e51" committed_config { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:16.166162 20525 ts_tablet_manager.cc:902] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Initiating tablet copy from peer e29339b99789441ebc37b6f327ad34ae (127.7.221.65:36593)
I20240502 02:04:16.166896 20525 tablet_copy_client.cc:308] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:36593
I20240502 02:04:16.170957 19440 tablet_copy_service.cc:140] P e29339b99789441ebc37b6f327ad34ae: Received BeginTabletCopySession request for tablet bc4cb3dc8c3340a88fbc50e175c68f49 from peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 ({username='slave'} at 127.7.221.67:34399)
I20240502 02:04:16.171108 19440 tablet_copy_service.cc:161] P e29339b99789441ebc37b6f327ad34ae: Beginning new tablet copy session on tablet bc4cb3dc8c3340a88fbc50e175c68f49 from peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 at {username='slave'} at 127.7.221.67:34399: session id = 5ba05ad7c35a41f4b1a0cb4bef0d6d15-bc4cb3dc8c3340a88fbc50e175c68f49
I20240502 02:04:16.172609 19440 tablet_copy_source_session.cc:205] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:16.175613 20525 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bc4cb3dc8c3340a88fbc50e175c68f49. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:16.176887 20137 heartbeater.cc:503] Master 127.7.221.126:43599 was elected leader, sending a full tablet report...
I20240502 02:04:16.181092 20525 tablet_copy_client.cc:783] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:16.181272 20525 tablet_copy_client.cc:647] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:16.182860 20525 tablet_copy_client.cc:523] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:16.184278 20525 tablet_bootstrap.cc:492] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Bootstrap starting.
I20240502 02:04:16.188402 20525 log.cc:824] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:16.190510 20525 tablet_bootstrap.cc:492] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:16.190804 20525 tablet_bootstrap.cc:492] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Bootstrap complete.
I20240502 02:04:16.191072 20525 ts_tablet_manager.cc:1367] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Time spent bootstrapping tablet: real 0.007s user 0.007s sys 0.001s
I20240502 02:04:16.193482 20525 raft_consensus.cc:348] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } }
I20240502 02:04:16.193675 20525 raft_consensus.cc:729] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 5ba05ad7c35a41f4b1a0cb4bef0d6d15, State: Initialized, Role: LEARNER
I20240502 02:04:16.193971 20525 consensus_queue.cc:260] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } }
I20240502 02:04:16.194751 20525 ts_tablet_manager.cc:1398] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Time spent starting tablet: real 0.004s user 0.004s sys 0.000s
I20240502 02:04:16.195541 19440 tablet_copy_service.cc:342] P e29339b99789441ebc37b6f327ad34ae: Request end of tablet copy session 5ba05ad7c35a41f4b1a0cb4bef0d6d15-bc4cb3dc8c3340a88fbc50e175c68f49 received from {username='slave'} at 127.7.221.67:34399
I20240502 02:04:16.195637 19440 tablet_copy_service.cc:433] P e29339b99789441ebc37b6f327ad34ae: ending tablet copy session 5ba05ad7c35a41f4b1a0cb4bef0d6d15-bc4cb3dc8c3340a88fbc50e175c68f49 on tablet bc4cb3dc8c3340a88fbc50e175c68f49 with peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15
I20240502 02:04:16.201061 20525 ts_tablet_manager.cc:902] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Initiating tablet copy from peer e29339b99789441ebc37b6f327ad34ae (127.7.221.65:36593)
I20240502 02:04:16.201467 20525 tablet_copy_client.cc:308] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:36593
I20240502 02:04:16.201900 19440 tablet_copy_service.cc:140] P e29339b99789441ebc37b6f327ad34ae: Received BeginTabletCopySession request for tablet c7aa1292397343e69abf05758d4eca7f from peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 ({username='slave'} at 127.7.221.67:34399)
I20240502 02:04:16.202004 19440 tablet_copy_service.cc:161] P e29339b99789441ebc37b6f327ad34ae: Beginning new tablet copy session on tablet c7aa1292397343e69abf05758d4eca7f from peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 at {username='slave'} at 127.7.221.67:34399: session id = 5ba05ad7c35a41f4b1a0cb4bef0d6d15-c7aa1292397343e69abf05758d4eca7f
I20240502 02:04:16.202723 19440 tablet_copy_source_session.cc:205] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:16.203444 20525 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c7aa1292397343e69abf05758d4eca7f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:16.205075 20525 tablet_copy_client.cc:783] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:16.205212 20525 tablet_copy_client.cc:647] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:16.206189 20525 tablet_copy_client.cc:523] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:16.207187 20525 tablet_bootstrap.cc:492] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Bootstrap starting.
I20240502 02:04:16.209924 20525 tablet_bootstrap.cc:492] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:16.210111 20525 tablet_bootstrap.cc:492] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Bootstrap complete.
I20240502 02:04:16.210224 20525 ts_tablet_manager.cc:1367] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.001s
I20240502 02:04:16.210755 20525 raft_consensus.cc:348] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } }
I20240502 02:04:16.210875 20525 raft_consensus.cc:729] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 5ba05ad7c35a41f4b1a0cb4bef0d6d15, State: Initialized, Role: LEARNER
I20240502 02:04:16.210981 20525 consensus_queue.cc:260] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } }
I20240502 02:04:16.211226 20525 ts_tablet_manager.cc:1398] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 02:04:16.211690 19440 tablet_copy_service.cc:342] P e29339b99789441ebc37b6f327ad34ae: Request end of tablet copy session 5ba05ad7c35a41f4b1a0cb4bef0d6d15-c7aa1292397343e69abf05758d4eca7f received from {username='slave'} at 127.7.221.67:34399
I20240502 02:04:16.211769 19440 tablet_copy_service.cc:433] P e29339b99789441ebc37b6f327ad34ae: ending tablet copy session 5ba05ad7c35a41f4b1a0cb4bef0d6d15-c7aa1292397343e69abf05758d4eca7f on tablet c7aa1292397343e69abf05758d4eca7f with peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15
I20240502 02:04:16.234279 20538 ts_tablet_manager.cc:902] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Initiating tablet copy from peer e29339b99789441ebc37b6f327ad34ae (127.7.221.65:36593)
I20240502 02:04:16.235083 20538 tablet_copy_client.cc:308] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:36593
I20240502 02:04:16.238277 19440 tablet_copy_service.cc:140] P e29339b99789441ebc37b6f327ad34ae: Received BeginTabletCopySession request for tablet 61234e24e53049f5be6c60dbbd170328 from peer 6ddcf3d8a67f4601b88c7cdab5fc6fec ({username='slave'} at 127.7.221.68:39767)
I20240502 02:04:16.238375 19440 tablet_copy_service.cc:161] P e29339b99789441ebc37b6f327ad34ae: Beginning new tablet copy session on tablet 61234e24e53049f5be6c60dbbd170328 from peer 6ddcf3d8a67f4601b88c7cdab5fc6fec at {username='slave'} at 127.7.221.68:39767: session id = 6ddcf3d8a67f4601b88c7cdab5fc6fec-61234e24e53049f5be6c60dbbd170328
I20240502 02:04:16.239140 19440 tablet_copy_source_session.cc:205] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:16.241995 20538 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 61234e24e53049f5be6c60dbbd170328. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:16.247536 20538 tablet_copy_client.cc:783] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:16.247731 20538 tablet_copy_client.cc:647] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:16.249070 20538 tablet_copy_client.cc:523] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:16.250422 20538 tablet_bootstrap.cc:492] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Bootstrap starting.
I20240502 02:04:16.254281 20538 log.cc:824] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:16.256381 20538 tablet_bootstrap.cc:492] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:16.256637 20538 tablet_bootstrap.cc:492] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Bootstrap complete.
I20240502 02:04:16.256875 20538 ts_tablet_manager.cc:1367] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Time spent bootstrapping tablet: real 0.007s user 0.006s sys 0.000s
I20240502 02:04:16.259251 20538 raft_consensus.cc:348] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: true } }
I20240502 02:04:16.259459 20538 raft_consensus.cc:729] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6ddcf3d8a67f4601b88c7cdab5fc6fec, State: Initialized, Role: LEARNER
I20240502 02:04:16.259769 20538 consensus_queue.cc:260] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: true } }
I20240502 02:04:16.260061 20545 ts_tablet_manager.cc:902] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Initiating tablet copy from peer e29339b99789441ebc37b6f327ad34ae (127.7.221.65:36593)
I20240502 02:04:16.260398 20270 heartbeater.cc:503] Master 127.7.221.126:43599 was elected leader, sending a full tablet report...
I20240502 02:04:16.260573 20538 ts_tablet_manager.cc:1398] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Time spent starting tablet: real 0.004s user 0.005s sys 0.000s
I20240502 02:04:16.260599 20545 tablet_copy_client.cc:308] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:36593
I20240502 02:04:16.261068 19440 tablet_copy_service.cc:140] P e29339b99789441ebc37b6f327ad34ae: Received BeginTabletCopySession request for tablet cdae5597120c4631b6c2ffa7176d60c9 from peer 6ddcf3d8a67f4601b88c7cdab5fc6fec ({username='slave'} at 127.7.221.68:39767)
I20240502 02:04:16.261190 19440 tablet_copy_service.cc:161] P e29339b99789441ebc37b6f327ad34ae: Beginning new tablet copy session on tablet cdae5597120c4631b6c2ffa7176d60c9 from peer 6ddcf3d8a67f4601b88c7cdab5fc6fec at {username='slave'} at 127.7.221.68:39767: session id = 6ddcf3d8a67f4601b88c7cdab5fc6fec-cdae5597120c4631b6c2ffa7176d60c9
I20240502 02:04:16.261523 19439 tablet_copy_service.cc:342] P e29339b99789441ebc37b6f327ad34ae: Request end of tablet copy session 6ddcf3d8a67f4601b88c7cdab5fc6fec-61234e24e53049f5be6c60dbbd170328 received from {username='slave'} at 127.7.221.68:39767
I20240502 02:04:16.261617 19439 tablet_copy_service.cc:433] P e29339b99789441ebc37b6f327ad34ae: ending tablet copy session 6ddcf3d8a67f4601b88c7cdab5fc6fec-61234e24e53049f5be6c60dbbd170328 on tablet 61234e24e53049f5be6c60dbbd170328 with peer 6ddcf3d8a67f4601b88c7cdab5fc6fec
I20240502 02:04:16.262221 19440 tablet_copy_source_session.cc:205] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:16.262930 20545 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cdae5597120c4631b6c2ffa7176d60c9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:16.264899 20545 tablet_copy_client.cc:783] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:16.265167 20545 tablet_copy_client.cc:647] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:16.266086 20545 tablet_copy_client.cc:523] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:16.267009 20545 tablet_bootstrap.cc:492] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Bootstrap starting.
I20240502 02:04:16.269737 20545 tablet_bootstrap.cc:492] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:16.269944 20545 tablet_bootstrap.cc:492] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Bootstrap complete.
I20240502 02:04:16.270079 20545 ts_tablet_manager.cc:1367] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 02:04:16.270545 20545 raft_consensus.cc:348] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: true } }
I20240502 02:04:16.270673 20545 raft_consensus.cc:729] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6ddcf3d8a67f4601b88c7cdab5fc6fec, State: Initialized, Role: LEARNER
I20240502 02:04:16.270779 20545 consensus_queue.cc:260] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: true } }
I20240502 02:04:16.271014 20545 ts_tablet_manager.cc:1398] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Time spent starting tablet: real 0.001s user 0.001s sys 0.001s
I20240502 02:04:16.271534 19440 tablet_copy_service.cc:342] P e29339b99789441ebc37b6f327ad34ae: Request end of tablet copy session 6ddcf3d8a67f4601b88c7cdab5fc6fec-cdae5597120c4631b6c2ffa7176d60c9 received from {username='slave'} at 127.7.221.68:39767
I20240502 02:04:16.271612 19440 tablet_copy_service.cc:433] P e29339b99789441ebc37b6f327ad34ae: ending tablet copy session 6ddcf3d8a67f4601b88c7cdab5fc6fec-cdae5597120c4631b6c2ffa7176d60c9 on tablet cdae5597120c4631b6c2ffa7176d60c9 with peer 6ddcf3d8a67f4601b88c7cdab5fc6fec
I20240502 02:04:16.369117 20545 ts_tablet_manager.cc:902] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Initiating tablet copy from peer d2faf854fabf42afa2c5355009726e51 (127.7.221.66:39613)
I20240502 02:04:16.369557 20545 tablet_copy_client.cc:308] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:39613
I20240502 02:04:16.373785 19576 tablet_copy_service.cc:140] P d2faf854fabf42afa2c5355009726e51: Received BeginTabletCopySession request for tablet 021ef17cfdbc4537b9b3fbb49b240af8 from peer 6ddcf3d8a67f4601b88c7cdab5fc6fec ({username='slave'} at 127.7.221.68:33335)
I20240502 02:04:16.373924 19576 tablet_copy_service.cc:161] P d2faf854fabf42afa2c5355009726e51: Beginning new tablet copy session on tablet 021ef17cfdbc4537b9b3fbb49b240af8 from peer 6ddcf3d8a67f4601b88c7cdab5fc6fec at {username='slave'} at 127.7.221.68:33335: session id = 6ddcf3d8a67f4601b88c7cdab5fc6fec-021ef17cfdbc4537b9b3fbb49b240af8
I20240502 02:04:16.375309 19576 tablet_copy_source_session.cc:205] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:16.376572 20545 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 021ef17cfdbc4537b9b3fbb49b240af8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:16.378861 20545 tablet_copy_client.cc:783] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:16.379144 20545 tablet_copy_client.cc:647] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:16.380517 20545 tablet_copy_client.cc:523] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:16.381711 20545 tablet_bootstrap.cc:492] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Bootstrap starting.
I20240502 02:04:16.384737 20525 ts_tablet_manager.cc:902] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Initiating tablet copy from peer d2faf854fabf42afa2c5355009726e51 (127.7.221.66:39613)
I20240502 02:04:16.385216 20525 tablet_copy_client.cc:308] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:39613
I20240502 02:04:16.385396 20545 tablet_bootstrap.cc:492] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:16.385680 20545 tablet_bootstrap.cc:492] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Bootstrap complete.
I20240502 02:04:16.385864 20545 ts_tablet_manager.cc:1367] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Time spent bootstrapping tablet: real 0.004s user 0.004s sys 0.000s
I20240502 02:04:16.386363 20545 raft_consensus.cc:348] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: true } }
I20240502 02:04:16.386484 20545 raft_consensus.cc:729] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6ddcf3d8a67f4601b88c7cdab5fc6fec, State: Initialized, Role: LEARNER
I20240502 02:04:16.386619 20545 consensus_queue.cc:260] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: true } }
I20240502 02:04:16.387405 20545 ts_tablet_manager.cc:1398] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 02:04:16.387920 19576 tablet_copy_service.cc:342] P d2faf854fabf42afa2c5355009726e51: Request end of tablet copy session 6ddcf3d8a67f4601b88c7cdab5fc6fec-021ef17cfdbc4537b9b3fbb49b240af8 received from {username='slave'} at 127.7.221.68:33335
I20240502 02:04:16.388010 19576 tablet_copy_service.cc:433] P d2faf854fabf42afa2c5355009726e51: ending tablet copy session 6ddcf3d8a67f4601b88c7cdab5fc6fec-021ef17cfdbc4537b9b3fbb49b240af8 on tablet 021ef17cfdbc4537b9b3fbb49b240af8 with peer 6ddcf3d8a67f4601b88c7cdab5fc6fec
I20240502 02:04:16.388939 19576 tablet_copy_service.cc:140] P d2faf854fabf42afa2c5355009726e51: Received BeginTabletCopySession request for tablet dc40ed8ee597423c9fc8993863b83d9e from peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 ({username='slave'} at 127.7.221.67:53679)
I20240502 02:04:16.389030 19576 tablet_copy_service.cc:161] P d2faf854fabf42afa2c5355009726e51: Beginning new tablet copy session on tablet dc40ed8ee597423c9fc8993863b83d9e from peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 at {username='slave'} at 127.7.221.67:53679: session id = 5ba05ad7c35a41f4b1a0cb4bef0d6d15-dc40ed8ee597423c9fc8993863b83d9e
I20240502 02:04:16.389961 19576 tablet_copy_source_session.cc:205] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:16.390770 20525 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dc40ed8ee597423c9fc8993863b83d9e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:16.392714 20525 tablet_copy_client.cc:783] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:16.392858 20525 tablet_copy_client.cc:647] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:16.393987 20525 tablet_copy_client.cc:523] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:16.395048 20525 tablet_bootstrap.cc:492] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Bootstrap starting.
I20240502 02:04:16.401273 20525 tablet_bootstrap.cc:492] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Bootstrap replayed 1/1 log segments. Stats: ops{read=3 overwritten=0 applied=3 ignored=0} inserts{seen=1 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:16.401486 20525 tablet_bootstrap.cc:492] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Bootstrap complete.
I20240502 02:04:16.401636 20525 ts_tablet_manager.cc:1367] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.002s
I20240502 02:04:16.402015 20525 raft_consensus.cc:348] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } }
I20240502 02:04:16.402107 20525 raft_consensus.cc:729] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 5ba05ad7c35a41f4b1a0cb4bef0d6d15, State: Initialized, Role: LEARNER
I20240502 02:04:16.402199 20525 consensus_queue.cc:260] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } }
I20240502 02:04:16.402429 20525 ts_tablet_manager.cc:1398] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:16.403056 19576 tablet_copy_service.cc:342] P d2faf854fabf42afa2c5355009726e51: Request end of tablet copy session 5ba05ad7c35a41f4b1a0cb4bef0d6d15-dc40ed8ee597423c9fc8993863b83d9e received from {username='slave'} at 127.7.221.67:53679
I20240502 02:04:16.403141 19576 tablet_copy_service.cc:433] P d2faf854fabf42afa2c5355009726e51: ending tablet copy session 5ba05ad7c35a41f4b1a0cb4bef0d6d15-dc40ed8ee597423c9fc8993863b83d9e on tablet dc40ed8ee597423c9fc8993863b83d9e with peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15
I20240502 02:04:16.408507 20604 ts_tablet_manager.cc:902] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae: Initiating tablet copy from peer d2faf854fabf42afa2c5355009726e51 (127.7.221.66:39613)
I20240502 02:04:16.409124 20604 tablet_copy_client.cc:308] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:39613
I20240502 02:04:16.412585 19576 tablet_copy_service.cc:140] P d2faf854fabf42afa2c5355009726e51: Received BeginTabletCopySession request for tablet 118dc310317a4c6b8fdbeb0a710e62b0 from peer e29339b99789441ebc37b6f327ad34ae ({username='slave'} at 127.7.221.65:35841)
I20240502 02:04:16.412680 19576 tablet_copy_service.cc:161] P d2faf854fabf42afa2c5355009726e51: Beginning new tablet copy session on tablet 118dc310317a4c6b8fdbeb0a710e62b0 from peer e29339b99789441ebc37b6f327ad34ae at {username='slave'} at 127.7.221.65:35841: session id = e29339b99789441ebc37b6f327ad34ae-118dc310317a4c6b8fdbeb0a710e62b0
I20240502 02:04:16.413527 19576 tablet_copy_source_session.cc:205] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:16.414582 20604 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 118dc310317a4c6b8fdbeb0a710e62b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:16.416807 20604 tablet_copy_client.cc:783] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:16.417011 20604 tablet_copy_client.cc:647] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:16.418438 20604 tablet_copy_client.cc:523] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:16.419865 20604 tablet_bootstrap.cc:492] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae: Bootstrap starting.
I20240502 02:04:16.424660 20604 tablet_bootstrap.cc:492] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae: Bootstrap replayed 1/1 log segments. Stats: ops{read=3 overwritten=0 applied=3 ignored=0} inserts{seen=1 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:16.424949 20604 tablet_bootstrap.cc:492] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae: Bootstrap complete.
I20240502 02:04:16.425141 20604 ts_tablet_manager.cc:1367] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240502 02:04:16.425539 20604 raft_consensus.cc:348] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: true } }
I20240502 02:04:16.425644 20604 raft_consensus.cc:729] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Initialized, Role: LEARNER
I20240502 02:04:16.425727 20604 consensus_queue.cc:260] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: true } }
I20240502 02:04:16.425968 20604 ts_tablet_manager.cc:1398] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:16.426578 19576 tablet_copy_service.cc:342] P d2faf854fabf42afa2c5355009726e51: Request end of tablet copy session e29339b99789441ebc37b6f327ad34ae-118dc310317a4c6b8fdbeb0a710e62b0 received from {username='slave'} at 127.7.221.65:35841
I20240502 02:04:16.426682 19576 tablet_copy_service.cc:433] P d2faf854fabf42afa2c5355009726e51: ending tablet copy session e29339b99789441ebc37b6f327ad34ae-118dc310317a4c6b8fdbeb0a710e62b0 on tablet 118dc310317a4c6b8fdbeb0a710e62b0 with peer e29339b99789441ebc37b6f327ad34ae
I20240502 02:04:16.435942 20525 ts_tablet_manager.cc:902] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Initiating tablet copy from peer d2faf854fabf42afa2c5355009726e51 (127.7.221.66:39613)
I20240502 02:04:16.436399 20525 tablet_copy_client.cc:308] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:39613
I20240502 02:04:16.436848 19576 tablet_copy_service.cc:140] P d2faf854fabf42afa2c5355009726e51: Received BeginTabletCopySession request for tablet d77eb55afc0e4dd796390ebb3961d0fc from peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 ({username='slave'} at 127.7.221.67:53679)
I20240502 02:04:16.436949 19576 tablet_copy_service.cc:161] P d2faf854fabf42afa2c5355009726e51: Beginning new tablet copy session on tablet d77eb55afc0e4dd796390ebb3961d0fc from peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 at {username='slave'} at 127.7.221.67:53679: session id = 5ba05ad7c35a41f4b1a0cb4bef0d6d15-d77eb55afc0e4dd796390ebb3961d0fc
I20240502 02:04:16.437610 19576 tablet_copy_source_session.cc:205] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:16.438347 20525 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d77eb55afc0e4dd796390ebb3961d0fc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:16.439958 20525 tablet_copy_client.cc:783] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:16.440091 20525 tablet_copy_client.cc:647] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:16.441114 20525 tablet_copy_client.cc:523] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:16.442010 20525 tablet_bootstrap.cc:492] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Bootstrap starting.
I20240502 02:04:16.447059 20525 tablet_bootstrap.cc:492] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:16.447288 20525 tablet_bootstrap.cc:492] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Bootstrap complete.
I20240502 02:04:16.447430 20525 ts_tablet_manager.cc:1367] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.005s
I20240502 02:04:16.447815 20525 raft_consensus.cc:348] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } }
I20240502 02:04:16.447916 20525 raft_consensus.cc:729] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 5ba05ad7c35a41f4b1a0cb4bef0d6d15, State: Initialized, Role: LEARNER
I20240502 02:04:16.448005 20525 consensus_queue.cc:260] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: true } }
I20240502 02:04:16.448241 20525 ts_tablet_manager.cc:1398] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:16.448721 19576 tablet_copy_service.cc:342] P d2faf854fabf42afa2c5355009726e51: Request end of tablet copy session 5ba05ad7c35a41f4b1a0cb4bef0d6d15-d77eb55afc0e4dd796390ebb3961d0fc received from {username='slave'} at 127.7.221.67:53679
I20240502 02:04:16.448794 19576 tablet_copy_service.cc:433] P d2faf854fabf42afa2c5355009726e51: ending tablet copy session 5ba05ad7c35a41f4b1a0cb4bef0d6d15-d77eb55afc0e4dd796390ebb3961d0fc on tablet d77eb55afc0e4dd796390ebb3961d0fc with peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15
I20240502 02:04:16.688395 20090 raft_consensus.cc:1203] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 02:04:16.761988 20090 raft_consensus.cc:1203] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 02:04:16.765158 20223 raft_consensus.cc:1203] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 02:04:16.790033 20223 raft_consensus.cc:1203] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 02:04:16.908063 20223 raft_consensus.cc:1203] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 02:04:16.928361 19419 raft_consensus.cc:1203] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [term 1 LEARNER]: Deduplicated request from leader. Original: 1.2->[1.3-1.3] Dedup: 1.3->[]
I20240502 02:04:16.952417 20090 raft_consensus.cc:1203] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 02:04:16.997617 20090 raft_consensus.cc:1203] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.2->[1.3-1.3] Dedup: 1.3->[]
I20240502 02:04:17.111770 20658 raft_consensus.cc:1050] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae: attempting to promote NON_VOTER 5ba05ad7c35a41f4b1a0cb4bef0d6d15 to VOTER
I20240502 02:04:17.112414 20658 consensus_queue.cc:237] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:17.113718 20090 raft_consensus.cc:1261] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Refusing update from remote peer e29339b99789441ebc37b6f327ad34ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 02:04:17.114192 20412 consensus_queue.cc:1035] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 02:04:17.117195 20412 raft_consensus.cc:2932] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } } }
I20240502 02:04:17.118942 20090 raft_consensus.cc:2932] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } } }
I20240502 02:04:17.119702 19276 catalog_manager.cc:5484] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae reported cstate change: config changed from index 2 to 3, 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:17.169823 20657 raft_consensus.cc:1050] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae: attempting to promote NON_VOTER 6ddcf3d8a67f4601b88c7cdab5fc6fec to VOTER
I20240502 02:04:17.170223 20657 consensus_queue.cc:237] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } }
I20240502 02:04:17.171617 20223 raft_consensus.cc:1261] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 LEARNER]: Refusing update from remote peer e29339b99789441ebc37b6f327ad34ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 02:04:17.172082 20658 consensus_queue.cc:1035] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 02:04:17.174801 20657 raft_consensus.cc:2932] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } } }
I20240502 02:04:17.175302 20223 raft_consensus.cc:2932] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } } }
I20240502 02:04:17.177301 19273 catalog_manager.cc:5484] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae reported cstate change: config changed from index 2 to 3, 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:17.342322 20412 raft_consensus.cc:1050] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae: attempting to promote NON_VOTER 6ddcf3d8a67f4601b88c7cdab5fc6fec to VOTER
I20240502 02:04:17.342730 20412 consensus_queue.cc:237] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } }
I20240502 02:04:17.343701 20223 raft_consensus.cc:1261] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 LEARNER]: Refusing update from remote peer e29339b99789441ebc37b6f327ad34ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 02:04:17.344209 20657 consensus_queue.cc:1035] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 02:04:17.346220 20412 raft_consensus.cc:2932] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } } }
I20240502 02:04:17.346738 20223 raft_consensus.cc:2932] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } } }
I20240502 02:04:17.348788 19276 catalog_manager.cc:5484] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae reported cstate change: config changed from index 2 to 3, 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:17.352263 19419 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "61234e24e53049f5be6c60dbbd170328"
dest_uuid: "e29339b99789441ebc37b6f327ad34ae"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:51118
I20240502 02:04:17.352406 19419 raft_consensus.cc:595] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:17.359423 20412 raft_consensus.cc:1050] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae: attempting to promote NON_VOTER 5ba05ad7c35a41f4b1a0cb4bef0d6d15 to VOTER
I20240502 02:04:17.359751 20412 consensus_queue.cc:237] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:17.360826 20090 raft_consensus.cc:1261] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Refusing update from remote peer e29339b99789441ebc37b6f327ad34ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 02:04:17.361218 20658 consensus_queue.cc:1035] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 02:04:17.363082 20412 raft_consensus.cc:2932] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } } }
I20240502 02:04:17.363590 20090 raft_consensus.cc:2932] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } } }
I20240502 02:04:17.365832 19273 catalog_manager.cc:5484] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae reported cstate change: config changed from index 2 to 3, 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:17.370699 19419 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "c7aa1292397343e69abf05758d4eca7f"
dest_uuid: "e29339b99789441ebc37b6f327ad34ae"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:51128
I20240502 02:04:17.370801 19419 raft_consensus.cc:595] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:17.386888 19419 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "cdae5597120c4631b6c2ffa7176d60c9"
dest_uuid: "e29339b99789441ebc37b6f327ad34ae"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:51144
I20240502 02:04:17.387007 19419 raft_consensus.cc:595] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:17.393498 20701 raft_consensus.cc:1050] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51: attempting to promote NON_VOTER 6ddcf3d8a67f4601b88c7cdab5fc6fec to VOTER
I20240502 02:04:17.394101 20701 consensus_queue.cc:237] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } }
I20240502 02:04:17.396060 20223 raft_consensus.cc:1261] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 LEARNER]: Refusing update from remote peer d2faf854fabf42afa2c5355009726e51: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 02:04:17.396497 20700 consensus_queue.cc:1035] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 02:04:17.398371 20701 raft_consensus.cc:2932] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } } }
I20240502 02:04:17.399209 20223 raft_consensus.cc:2932] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } } }
I20240502 02:04:17.401172 19276 catalog_manager.cc:5484] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 reported cstate change: config changed from index 2 to 3, 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "d2faf854fabf42afa2c5355009726e51" committed_config { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:17.406088 19419 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "bc4cb3dc8c3340a88fbc50e175c68f49"
dest_uuid: "e29339b99789441ebc37b6f327ad34ae"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:51154
I20240502 02:04:17.406203 19419 raft_consensus.cc:595] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:17.468102 20578 raft_consensus.cc:1050] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51: attempting to promote NON_VOTER e29339b99789441ebc37b6f327ad34ae to VOTER
I20240502 02:04:17.468439 20578 consensus_queue.cc:237] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: false } }
I20240502 02:04:17.469482 19419 raft_consensus.cc:1261] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [term 1 LEARNER]: Refusing update from remote peer d2faf854fabf42afa2c5355009726e51: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:17.469883 20578 consensus_queue.cc:1035] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:17.471644 20700 raft_consensus.cc:2932] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index 3 to 4, e29339b99789441ebc37b6f327ad34ae (127.7.221.65) changed from NON_VOTER to VOTER. New config: { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: false } } }
I20240502 02:04:17.472074 19419 raft_consensus.cc:2932] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, e29339b99789441ebc37b6f327ad34ae (127.7.221.65) changed from NON_VOTER to VOTER. New config: { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: false } } }
I20240502 02:04:17.474318 19273 catalog_manager.cc:5484] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 reported cstate change: config changed from index 3 to 4, e29339b99789441ebc37b6f327ad34ae (127.7.221.65) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "d2faf854fabf42afa2c5355009726e51" committed_config { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:17.510203 20700 raft_consensus.cc:1050] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51: attempting to promote NON_VOTER 5ba05ad7c35a41f4b1a0cb4bef0d6d15 to VOTER
I20240502 02:04:17.510535 20700 consensus_queue.cc:237] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:17.511626 20090 raft_consensus.cc:1261] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Refusing update from remote peer d2faf854fabf42afa2c5355009726e51: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 02:04:17.512014 20700 consensus_queue.cc:1035] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 02:04:17.513614 20701 raft_consensus.cc:2932] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } } }
I20240502 02:04:17.514025 20090 raft_consensus.cc:2932] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } } }
I20240502 02:04:17.516278 19276 catalog_manager.cc:5484] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 reported cstate change: config changed from index 2 to 3, 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "d2faf854fabf42afa2c5355009726e51" committed_config { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } } }
I20240502 02:04:17.590862 20701 raft_consensus.cc:1050] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51: attempting to promote NON_VOTER 5ba05ad7c35a41f4b1a0cb4bef0d6d15 to VOTER
I20240502 02:04:17.591182 20701 consensus_queue.cc:237] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:17.592175 20090 raft_consensus.cc:1261] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 LEARNER]: Refusing update from remote peer d2faf854fabf42afa2c5355009726e51: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:17.592516 20701 consensus_queue.cc:1035] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:17.594163 20578 raft_consensus.cc:2932] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index 3 to 4, 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) changed from NON_VOTER to VOTER. New config: { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } } }
I20240502 02:04:17.594646 20090 raft_consensus.cc:2932] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) changed from NON_VOTER to VOTER. New config: { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } } }
I20240502 02:04:17.596923 19273 catalog_manager.cc:5484] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 reported cstate change: config changed from index 3 to 4, 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "d2faf854fabf42afa2c5355009726e51" committed_config { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:17.617298 19555 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "dc40ed8ee597423c9fc8993863b83d9e"
dest_uuid: "d2faf854fabf42afa2c5355009726e51"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40474
I20240502 02:04:17.617470 19555 raft_consensus.cc:595] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:17.627259 20412 raft_consensus.cc:979] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae: : Instructing follower 6ddcf3d8a67f4601b88c7cdab5fc6fec to start an election
I20240502 02:04:17.627372 20658 raft_consensus.cc:1067] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Signalling peer 6ddcf3d8a67f4601b88c7cdab5fc6fec to start an election
I20240502 02:04:17.627599 19419 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "61234e24e53049f5be6c60dbbd170328"
dest_uuid: "e29339b99789441ebc37b6f327ad34ae"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:51174
I20240502 02:04:17.627727 19419 raft_consensus.cc:595] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:17.628024 20223 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "61234e24e53049f5be6c60dbbd170328"
dest_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec"
from {username='slave'} at 127.7.221.65:37805
I20240502 02:04:17.628206 20223 raft_consensus.cc:482] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 02:04:17.628312 20223 raft_consensus.cc:3037] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:17.629473 20223 raft_consensus.cc:504] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } }
I20240502 02:04:17.630272 20223 leader_election.cc:288] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [CANDIDATE]: Term 2 election: Requested vote from peers e29339b99789441ebc37b6f327ad34ae (127.7.221.65:36593)
I20240502 02:04:17.630704 19419 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "61234e24e53049f5be6c60dbbd170328" candidate_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" candidate_term: 2 candidate_status { last_received { term: 1 index: 3 } } ignore_live_leader: true dest_uuid: "e29339b99789441ebc37b6f327ad34ae"
I20240502 02:04:17.630853 19419 raft_consensus.cc:3032] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Stepping down as leader of term 1
I20240502 02:04:17.630901 19419 raft_consensus.cc:729] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Becoming Follower/Learner. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Running, Role: LEADER
I20240502 02:04:17.631054 19419 consensus_queue.cc:260] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } }
I20240502 02:04:17.631218 19419 raft_consensus.cc:3037] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:17.632061 19419 raft_consensus.cc:2454] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6ddcf3d8a67f4601b88c7cdab5fc6fec in term 2.
I20240502 02:04:17.632513 20157 leader_election.cc:302] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6ddcf3d8a67f4601b88c7cdab5fc6fec, e29339b99789441ebc37b6f327ad34ae; no voters:
I20240502 02:04:17.632776 20546 raft_consensus.cc:2781] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 2 FOLLOWER]: Leader election won for term 2
I20240502 02:04:17.632944 20546 raft_consensus.cc:686] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 2 LEADER]: Becoming Leader. State: Replica: 6ddcf3d8a67f4601b88c7cdab5fc6fec, State: Running, Role: LEADER
I20240502 02:04:17.633222 20546 consensus_queue.cc:237] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } }
I20240502 02:04:17.633965 20658 raft_consensus.cc:979] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae: : Instructing follower 5ba05ad7c35a41f4b1a0cb4bef0d6d15 to start an election
I20240502 02:04:17.634037 20658 raft_consensus.cc:1067] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Signalling peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 to start an election
I20240502 02:04:17.634500 20090 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "c7aa1292397343e69abf05758d4eca7f"
dest_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15"
from {username='slave'} at 127.7.221.65:59317
I20240502 02:04:17.634714 20090 raft_consensus.cc:482] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 02:04:17.634815 20090 raft_consensus.cc:3037] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:17.635705 19276 catalog_manager.cc:5484] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec reported cstate change: term changed from 1 to 2, leader changed from e29339b99789441ebc37b6f327ad34ae (127.7.221.65) to 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68). New cstate: current_term: 2 leader_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" committed_config { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:17.635792 20090 raft_consensus.cc:504] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:17.636521 20090 leader_election.cc:288] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [CANDIDATE]: Term 2 election: Requested vote from peers e29339b99789441ebc37b6f327ad34ae (127.7.221.65:36593)
I20240502 02:04:17.637013 19419 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c7aa1292397343e69abf05758d4eca7f" candidate_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" candidate_term: 2 candidate_status { last_received { term: 1 index: 3 } } ignore_live_leader: true dest_uuid: "e29339b99789441ebc37b6f327ad34ae"
I20240502 02:04:17.637110 19419 raft_consensus.cc:3032] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Stepping down as leader of term 1
I20240502 02:04:17.637152 19419 raft_consensus.cc:729] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Becoming Follower/Learner. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Running, Role: LEADER
I20240502 02:04:17.637281 19419 consensus_queue.cc:260] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:17.637395 19419 raft_consensus.cc:3037] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:17.638154 19419 raft_consensus.cc:2454] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5ba05ad7c35a41f4b1a0cb4bef0d6d15 in term 2.
I20240502 02:04:17.638459 20024 leader_election.cc:302] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 5ba05ad7c35a41f4b1a0cb4bef0d6d15, e29339b99789441ebc37b6f327ad34ae; no voters:
I20240502 02:04:17.638738 20532 raft_consensus.cc:2781] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 02:04:17.638960 20532 raft_consensus.cc:686] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 LEADER]: Becoming Leader. State: Replica: 5ba05ad7c35a41f4b1a0cb4bef0d6d15, State: Running, Role: LEADER
I20240502 02:04:17.639212 20532 consensus_queue.cc:237] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:17.641821 19555 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "021ef17cfdbc4537b9b3fbb49b240af8"
dest_uuid: "d2faf854fabf42afa2c5355009726e51"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40488
I20240502 02:04:17.641789 19276 catalog_manager.cc:5484] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 reported cstate change: term changed from 1 to 2, leader changed from e29339b99789441ebc37b6f327ad34ae (127.7.221.65) to 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67). New cstate: current_term: 2 leader_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" committed_config { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:17.641935 19555 raft_consensus.cc:595] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Received request to transfer leadership
W20240502 02:04:17.645658 19263 catalog_manager.cc:5014] ChangeConfig:REMOVE_PEER RPC for tablet 61234e24e53049f5be6c60dbbd170328 with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER failed with leader 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68:36343) due to error UNKNOWN_ERROR; will retry: Illegal state: Leader has not yet committed an operation in its own term
I20240502 02:04:17.658967 19555 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "118dc310317a4c6b8fdbeb0a710e62b0"
dest_uuid: "d2faf854fabf42afa2c5355009726e51"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40498
I20240502 02:04:17.659092 19555 raft_consensus.cc:595] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:17.668967 19419 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "cdae5597120c4631b6c2ffa7176d60c9"
dest_uuid: "e29339b99789441ebc37b6f327ad34ae"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:51194
I20240502 02:04:17.669090 19419 raft_consensus.cc:595] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:17.678822 19555 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "d77eb55afc0e4dd796390ebb3961d0fc"
dest_uuid: "d2faf854fabf42afa2c5355009726e51"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40514
I20240502 02:04:17.678946 19555 raft_consensus.cc:595] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:17.689307 19419 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "bc4cb3dc8c3340a88fbc50e175c68f49"
dest_uuid: "e29339b99789441ebc37b6f327ad34ae"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:51224
I20240502 02:04:17.689424 19419 raft_consensus.cc:595] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:17.755291 20658 raft_consensus.cc:979] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae: : Instructing follower 6ddcf3d8a67f4601b88c7cdab5fc6fec to start an election
I20240502 02:04:17.755405 20658 raft_consensus.cc:1067] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Signalling peer 6ddcf3d8a67f4601b88c7cdab5fc6fec to start an election
I20240502 02:04:17.755868 20223 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "cdae5597120c4631b6c2ffa7176d60c9"
dest_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec"
from {username='slave'} at 127.7.221.65:37805
I20240502 02:04:17.756007 20223 raft_consensus.cc:482] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 02:04:17.756075 20223 raft_consensus.cc:3037] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:17.756968 20223 raft_consensus.cc:504] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } }
I20240502 02:04:17.757269 20223 leader_election.cc:288] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [CANDIDATE]: Term 2 election: Requested vote from peers e29339b99789441ebc37b6f327ad34ae (127.7.221.65:36593)
I20240502 02:04:17.757648 19419 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cdae5597120c4631b6c2ffa7176d60c9" candidate_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" candidate_term: 2 candidate_status { last_received { term: 1 index: 3 } } ignore_live_leader: true dest_uuid: "e29339b99789441ebc37b6f327ad34ae"
I20240502 02:04:17.757772 19419 raft_consensus.cc:3032] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Stepping down as leader of term 1
I20240502 02:04:17.757822 19419 raft_consensus.cc:729] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Becoming Follower/Learner. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Running, Role: LEADER
I20240502 02:04:17.757947 19419 consensus_queue.cc:260] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } }
I20240502 02:04:17.758059 19419 raft_consensus.cc:3037] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:17.758764 19419 raft_consensus.cc:2454] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6ddcf3d8a67f4601b88c7cdab5fc6fec in term 2.
I20240502 02:04:17.759014 20157 leader_election.cc:302] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6ddcf3d8a67f4601b88c7cdab5fc6fec, e29339b99789441ebc37b6f327ad34ae; no voters:
I20240502 02:04:17.759192 20546 raft_consensus.cc:2781] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 2 FOLLOWER]: Leader election won for term 2
I20240502 02:04:17.759287 20546 raft_consensus.cc:686] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 2 LEADER]: Becoming Leader. State: Replica: 6ddcf3d8a67f4601b88c7cdab5fc6fec, State: Running, Role: LEADER
I20240502 02:04:17.759434 20546 consensus_queue.cc:237] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } }
I20240502 02:04:17.760788 19276 catalog_manager.cc:5484] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec reported cstate change: term changed from 1 to 2, leader changed from e29339b99789441ebc37b6f327ad34ae (127.7.221.65) to 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68). New cstate: current_term: 2 leader_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" committed_config { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:17.830087 20578 raft_consensus.cc:979] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51: : Instructing follower 6ddcf3d8a67f4601b88c7cdab5fc6fec to start an election
I20240502 02:04:17.830214 20578 raft_consensus.cc:1067] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Signalling peer 6ddcf3d8a67f4601b88c7cdab5fc6fec to start an election
I20240502 02:04:17.830765 20223 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "021ef17cfdbc4537b9b3fbb49b240af8"
dest_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec"
from {username='slave'} at 127.7.221.66:49595
I20240502 02:04:17.830920 20223 raft_consensus.cc:482] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 02:04:17.830986 20223 raft_consensus.cc:3037] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:17.831756 20223 raft_consensus.cc:504] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } }
I20240502 02:04:17.832144 20223 leader_election.cc:288] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [CANDIDATE]: Term 2 election: Requested vote from peers d2faf854fabf42afa2c5355009726e51 (127.7.221.66:39613)
I20240502 02:04:17.832598 19555 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "021ef17cfdbc4537b9b3fbb49b240af8" candidate_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" candidate_term: 2 candidate_status { last_received { term: 1 index: 3 } } ignore_live_leader: true dest_uuid: "d2faf854fabf42afa2c5355009726e51"
I20240502 02:04:17.832756 19555 raft_consensus.cc:3032] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 02:04:17.832806 19555 raft_consensus.cc:729] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Running, Role: LEADER
I20240502 02:04:17.832976 19555 consensus_queue.cc:260] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } }
I20240502 02:04:17.833127 19555 raft_consensus.cc:3037] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:17.833971 19555 raft_consensus.cc:2454] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6ddcf3d8a67f4601b88c7cdab5fc6fec in term 2.
I20240502 02:04:17.834302 20156 leader_election.cc:302] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 6ddcf3d8a67f4601b88c7cdab5fc6fec, d2faf854fabf42afa2c5355009726e51; no voters:
I20240502 02:04:17.834486 20546 raft_consensus.cc:2781] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 2 FOLLOWER]: Leader election won for term 2
I20240502 02:04:17.834610 20546 raft_consensus.cc:686] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 2 LEADER]: Becoming Leader. State: Replica: 6ddcf3d8a67f4601b88c7cdab5fc6fec, State: Running, Role: LEADER
I20240502 02:04:17.834764 20546 consensus_queue.cc:237] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } }
I20240502 02:04:17.836182 19276 catalog_manager.cc:5484] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec reported cstate change: term changed from 1 to 2, leader changed from d2faf854fabf42afa2c5355009726e51 (127.7.221.66) to 6ddcf3d8a67f4601b88c7cdab5fc6fec (127.7.221.68). New cstate: current_term: 2 leader_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" committed_config { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:17.853381 20412 raft_consensus.cc:979] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae: : Instructing follower 5ba05ad7c35a41f4b1a0cb4bef0d6d15 to start an election
I20240502 02:04:17.853510 20412 raft_consensus.cc:1067] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Signalling peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 to start an election
I20240502 02:04:17.853974 20090 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "bc4cb3dc8c3340a88fbc50e175c68f49"
dest_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15"
from {username='slave'} at 127.7.221.65:59317
I20240502 02:04:17.854116 20090 raft_consensus.cc:482] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 02:04:17.854182 20090 raft_consensus.cc:3037] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:17.855015 20090 raft_consensus.cc:504] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:17.855342 20090 leader_election.cc:288] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [CANDIDATE]: Term 2 election: Requested vote from peers e29339b99789441ebc37b6f327ad34ae (127.7.221.65:36593)
I20240502 02:04:17.855741 19419 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bc4cb3dc8c3340a88fbc50e175c68f49" candidate_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" candidate_term: 2 candidate_status { last_received { term: 1 index: 3 } } ignore_live_leader: true dest_uuid: "e29339b99789441ebc37b6f327ad34ae"
I20240502 02:04:17.855870 19419 raft_consensus.cc:3032] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Stepping down as leader of term 1
I20240502 02:04:17.855928 19419 raft_consensus.cc:729] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 1 LEADER]: Becoming Follower/Learner. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Running, Role: LEADER
I20240502 02:04:17.856099 19419 consensus_queue.cc:260] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:17.856236 19419 raft_consensus.cc:3037] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:17.857031 19419 raft_consensus.cc:2454] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5ba05ad7c35a41f4b1a0cb4bef0d6d15 in term 2.
I20240502 02:04:17.857306 20024 leader_election.cc:302] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 5ba05ad7c35a41f4b1a0cb4bef0d6d15, e29339b99789441ebc37b6f327ad34ae; no voters:
I20240502 02:04:17.857524 20532 raft_consensus.cc:2781] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 02:04:17.857614 20532 raft_consensus.cc:686] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 LEADER]: Becoming Leader. State: Replica: 5ba05ad7c35a41f4b1a0cb4bef0d6d15, State: Running, Role: LEADER
I20240502 02:04:17.857760 20532 consensus_queue.cc:237] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:17.859232 19276 catalog_manager.cc:5484] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 reported cstate change: term changed from 1 to 2, leader changed from e29339b99789441ebc37b6f327ad34ae (127.7.221.65) to 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67). New cstate: current_term: 2 leader_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" committed_config { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:17.900811 19555 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "dc40ed8ee597423c9fc8993863b83d9e"
dest_uuid: "d2faf854fabf42afa2c5355009726e51"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40532
I20240502 02:04:17.900944 19555 raft_consensus.cc:595] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:17.926926 19555 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "118dc310317a4c6b8fdbeb0a710e62b0"
dest_uuid: "d2faf854fabf42afa2c5355009726e51"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40544
I20240502 02:04:17.927055 19555 raft_consensus.cc:595] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:17.942263 19555 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "d77eb55afc0e4dd796390ebb3961d0fc"
dest_uuid: "d2faf854fabf42afa2c5355009726e51"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40554
I20240502 02:04:17.942384 19555 raft_consensus.cc:595] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:17.983888 20700 raft_consensus.cc:979] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51: : Instructing follower 5ba05ad7c35a41f4b1a0cb4bef0d6d15 to start an election
I20240502 02:04:17.984009 20700 raft_consensus.cc:1067] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Signalling peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 to start an election
I20240502 02:04:17.984577 20089 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "d77eb55afc0e4dd796390ebb3961d0fc"
dest_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15"
from {username='slave'} at 127.7.221.66:33023
I20240502 02:04:17.984746 20089 raft_consensus.cc:482] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 02:04:17.984825 20089 raft_consensus.cc:3037] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:17.985713 20089 raft_consensus.cc:504] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:17.986044 20089 leader_election.cc:288] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [CANDIDATE]: Term 2 election: Requested vote from peers d2faf854fabf42afa2c5355009726e51 (127.7.221.66:39613)
I20240502 02:04:17.986452 19555 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d77eb55afc0e4dd796390ebb3961d0fc" candidate_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" candidate_term: 2 candidate_status { last_received { term: 1 index: 3 } } ignore_live_leader: true dest_uuid: "d2faf854fabf42afa2c5355009726e51"
I20240502 02:04:17.986572 19555 raft_consensus.cc:3032] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 02:04:17.986660 19555 raft_consensus.cc:729] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Running, Role: LEADER
I20240502 02:04:17.986822 19555 consensus_queue.cc:260] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:17.986969 19555 raft_consensus.cc:3037] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:17.987707 19555 raft_consensus.cc:2454] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5ba05ad7c35a41f4b1a0cb4bef0d6d15 in term 2.
I20240502 02:04:17.987978 20023 leader_election.cc:302] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 5ba05ad7c35a41f4b1a0cb4bef0d6d15, d2faf854fabf42afa2c5355009726e51; no voters:
I20240502 02:04:17.988143 20532 raft_consensus.cc:2781] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 02:04:17.988226 20532 raft_consensus.cc:686] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 LEADER]: Becoming Leader. State: Replica: 5ba05ad7c35a41f4b1a0cb4bef0d6d15, State: Running, Role: LEADER
I20240502 02:04:17.988366 20532 consensus_queue.cc:237] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:17.989789 19276 catalog_manager.cc:5484] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 reported cstate change: term changed from 1 to 2, leader changed from d2faf854fabf42afa2c5355009726e51 (127.7.221.66) to 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67). New cstate: current_term: 2 leader_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" committed_config { opid_index: 3 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:18.031566 19419 raft_consensus.cc:1261] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Refusing update from remote peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 2 index: 4. (index mismatch)
I20240502 02:04:18.032138 20532 consensus_queue.cc:1035] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:18.073215 20578 raft_consensus.cc:979] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51: : Instructing follower e29339b99789441ebc37b6f327ad34ae to start an election
I20240502 02:04:18.073323 20578 raft_consensus.cc:1067] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Signalling peer e29339b99789441ebc37b6f327ad34ae to start an election
I20240502 02:04:18.073783 19419 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "118dc310317a4c6b8fdbeb0a710e62b0"
dest_uuid: "e29339b99789441ebc37b6f327ad34ae"
from {username='slave'} at 127.7.221.66:40741
I20240502 02:04:18.073936 19419 raft_consensus.cc:482] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 02:04:18.073998 19419 raft_consensus.cc:3037] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:18.074860 19419 raft_consensus.cc:504] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: false } }
I20240502 02:04:18.075423 19419 leader_election.cc:288] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [CANDIDATE]: Term 2 election: Requested vote from peers d2faf854fabf42afa2c5355009726e51 (127.7.221.66:39613)
I20240502 02:04:18.075820 19555 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "118dc310317a4c6b8fdbeb0a710e62b0" candidate_uuid: "e29339b99789441ebc37b6f327ad34ae" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: true dest_uuid: "d2faf854fabf42afa2c5355009726e51"
I20240502 02:04:18.075942 19555 raft_consensus.cc:3032] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 02:04:18.075996 19555 raft_consensus.cc:729] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Running, Role: LEADER
I20240502 02:04:18.076148 19555 consensus_queue.cc:260] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: false } }
I20240502 02:04:18.076272 19555 raft_consensus.cc:3037] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:18.077011 19555 raft_consensus.cc:2454] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e29339b99789441ebc37b6f327ad34ae in term 2.
I20240502 02:04:18.077311 19353 leader_election.cc:302] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: d2faf854fabf42afa2c5355009726e51, e29339b99789441ebc37b6f327ad34ae; no voters:
I20240502 02:04:18.077483 20412 raft_consensus.cc:2781] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Leader election won for term 2
I20240502 02:04:18.077575 20412 raft_consensus.cc:686] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [term 2 LEADER]: Becoming Leader. State: Replica: e29339b99789441ebc37b6f327ad34ae, State: Running, Role: LEADER
I20240502 02:04:18.077713 20412 consensus_queue.cc:237] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: false } }
I20240502 02:04:18.079056 19273 catalog_manager.cc:5484] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae reported cstate change: term changed from 1 to 2, leader changed from d2faf854fabf42afa2c5355009726e51 (127.7.221.66) to e29339b99789441ebc37b6f327ad34ae (127.7.221.65). New cstate: current_term: 2 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:18.159854 19555 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "dc40ed8ee597423c9fc8993863b83d9e"
dest_uuid: "d2faf854fabf42afa2c5355009726e51"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40580
I20240502 02:04:18.159996 19555 raft_consensus.cc:595] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:18.203714 20700 raft_consensus.cc:979] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51: : Instructing follower 5ba05ad7c35a41f4b1a0cb4bef0d6d15 to start an election
I20240502 02:04:18.203806 20700 raft_consensus.cc:1067] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Signalling peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15 to start an election
I20240502 02:04:18.204257 20089 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "dc40ed8ee597423c9fc8993863b83d9e"
dest_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15"
from {username='slave'} at 127.7.221.66:33023
I20240502 02:04:18.204370 20089 raft_consensus.cc:482] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 02:04:18.204416 20089 raft_consensus.cc:3037] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:18.205181 20089 raft_consensus.cc:504] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:18.205439 20089 leader_election.cc:288] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [CANDIDATE]: Term 2 election: Requested vote from peers d2faf854fabf42afa2c5355009726e51 (127.7.221.66:39613)
I20240502 02:04:18.205804 19555 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dc40ed8ee597423c9fc8993863b83d9e" candidate_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: true dest_uuid: "d2faf854fabf42afa2c5355009726e51"
I20240502 02:04:18.205940 19555 raft_consensus.cc:3032] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 02:04:18.205987 19555 raft_consensus.cc:729] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: d2faf854fabf42afa2c5355009726e51, State: Running, Role: LEADER
I20240502 02:04:18.206136 19555 consensus_queue.cc:260] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:18.206271 19555 raft_consensus.cc:3037] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:18.207033 19555 raft_consensus.cc:2454] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5ba05ad7c35a41f4b1a0cb4bef0d6d15 in term 2.
I20240502 02:04:18.207298 20023 leader_election.cc:302] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 5ba05ad7c35a41f4b1a0cb4bef0d6d15, d2faf854fabf42afa2c5355009726e51; no voters:
I20240502 02:04:18.207487 20857 raft_consensus.cc:2781] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 02:04:18.207579 20857 raft_consensus.cc:686] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 LEADER]: Becoming Leader. State: Replica: 5ba05ad7c35a41f4b1a0cb4bef0d6d15, State: Running, Role: LEADER
I20240502 02:04:18.207715 20857 consensus_queue.cc:237] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:18.209015 19273 catalog_manager.cc:5484] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 reported cstate change: term changed from 1 to 2, leader changed from d2faf854fabf42afa2c5355009726e51 (127.7.221.66) to 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67). New cstate: current_term: 2 leader_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" committed_config { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:18.215322 19419 raft_consensus.cc:1261] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Refusing update from remote peer 6ddcf3d8a67f4601b88c7cdab5fc6fec: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 2 index: 4. (index mismatch)
I20240502 02:04:18.215745 20546 consensus_queue.cc:1035] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [LEADER]: Connected to new peer: Peer: permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:18.236177 19419 raft_consensus.cc:1261] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Refusing update from remote peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 2 index: 4. (index mismatch)
I20240502 02:04:18.236615 20857 consensus_queue.cc:1035] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:18.278353 20089 consensus_queue.cc:237] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 2.4, Last appended by leader: 3, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:18.279002 20857 raft_consensus.cc:2932] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 LEADER]: Committing config change with OpId 2.5: config changed from index 3 to 5, VOTER e29339b99789441ebc37b6f327ad34ae (127.7.221.65) evicted. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } } }
I20240502 02:04:18.280282 19260 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c7aa1292397343e69abf05758d4eca7f with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 02:04:18.281086 19273 catalog_manager.cc:5484] T c7aa1292397343e69abf05758d4eca7f P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 reported cstate change: config changed from index 3 to 5, VOTER e29339b99789441ebc37b6f327ad34ae (127.7.221.65) evicted. New cstate: current_term: 2 leader_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:18.283571 19397 tablet_service.cc:1514] Processing DeleteTablet for tablet c7aa1292397343e69abf05758d4eca7f with delete_type TABLET_DATA_TOMBSTONED (TS e29339b99789441ebc37b6f327ad34ae not found in new config with opid_index 5) from {username='slave'} at 127.0.0.1:46998
I20240502 02:04:18.284020 21013 tablet_replica.cc:330] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae: stopping tablet replica
I20240502 02:04:18.284327 21013 raft_consensus.cc:2229] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:18.284477 21013 raft_consensus.cc:2258] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:18.285305 21013 ts_tablet_manager.cc:1875] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:18.287086 21013 ts_tablet_manager.cc:1888] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.4
I20240502 02:04:18.287158 21013 log.cc:1196] T c7aa1292397343e69abf05758d4eca7f P e29339b99789441ebc37b6f327ad34ae: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/c7aa1292397343e69abf05758d4eca7f
I20240502 02:04:18.287663 19262 catalog_manager.cc:4830] TS e29339b99789441ebc37b6f327ad34ae (127.7.221.65:36593): tablet c7aa1292397343e69abf05758d4eca7f (table rebalance_test_table_2 [id=e95f8116d31f4b4b8526822b427924ee]) successfully deleted
I20240502 02:04:18.296475 20223 consensus_queue.cc:237] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 2.4, Last appended by leader: 3, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } }
I20240502 02:04:18.297086 20851 raft_consensus.cc:2932] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 2 LEADER]: Committing config change with OpId 2.5: config changed from index 3 to 5, VOTER e29339b99789441ebc37b6f327ad34ae (127.7.221.65) evicted. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } } }
I20240502 02:04:18.298252 19263 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 61234e24e53049f5be6c60dbbd170328 with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 02:04:18.299139 19273 catalog_manager.cc:5484] T 61234e24e53049f5be6c60dbbd170328 P 6ddcf3d8a67f4601b88c7cdab5fc6fec reported cstate change: config changed from index 3 to 5, VOTER e29339b99789441ebc37b6f327ad34ae (127.7.221.65) evicted. New cstate: current_term: 2 leader_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:18.301187 19397 tablet_service.cc:1514] Processing DeleteTablet for tablet 61234e24e53049f5be6c60dbbd170328 with delete_type TABLET_DATA_TOMBSTONED (TS e29339b99789441ebc37b6f327ad34ae not found in new config with opid_index 5) from {username='slave'} at 127.0.0.1:46998
I20240502 02:04:18.301404 21013 tablet_replica.cc:330] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae: stopping tablet replica
I20240502 02:04:18.301548 21013 raft_consensus.cc:2229] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:18.301646 21013 raft_consensus.cc:2258] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:18.302151 21013 ts_tablet_manager.cc:1875] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:18.303719 21013 ts_tablet_manager.cc:1888] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.4
I20240502 02:04:18.303778 21013 log.cc:1196] T 61234e24e53049f5be6c60dbbd170328 P e29339b99789441ebc37b6f327ad34ae: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/61234e24e53049f5be6c60dbbd170328
I20240502 02:04:18.304239 19262 catalog_manager.cc:4830] TS e29339b99789441ebc37b6f327ad34ae (127.7.221.65:36593): tablet 61234e24e53049f5be6c60dbbd170328 (table rebalance_test_table_2 [id=e95f8116d31f4b4b8526822b427924ee]) successfully deleted
I20240502 02:04:18.380807 19419 raft_consensus.cc:1261] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Refusing update from remote peer 6ddcf3d8a67f4601b88c7cdab5fc6fec: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 2 index: 4. (index mismatch)
I20240502 02:04:18.381325 20546 consensus_queue.cc:1035] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [LEADER]: Connected to new peer: Peer: permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:18.403414 20223 consensus_queue.cc:237] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 2.4, Last appended by leader: 3, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } }
I20240502 02:04:18.404104 20546 raft_consensus.cc:2932] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 2 LEADER]: Committing config change with OpId 2.5: config changed from index 3 to 5, VOTER e29339b99789441ebc37b6f327ad34ae (127.7.221.65) evicted. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } } }
I20240502 02:04:18.405611 19263 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cdae5597120c4631b6c2ffa7176d60c9 with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 02:04:18.406595 19276 catalog_manager.cc:5484] T cdae5597120c4631b6c2ffa7176d60c9 P 6ddcf3d8a67f4601b88c7cdab5fc6fec reported cstate change: config changed from index 3 to 5, VOTER e29339b99789441ebc37b6f327ad34ae (127.7.221.65) evicted. New cstate: current_term: 2 leader_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:18.408993 19397 tablet_service.cc:1514] Processing DeleteTablet for tablet cdae5597120c4631b6c2ffa7176d60c9 with delete_type TABLET_DATA_TOMBSTONED (TS e29339b99789441ebc37b6f327ad34ae not found in new config with opid_index 5) from {username='slave'} at 127.0.0.1:46998
I20240502 02:04:18.409230 21013 tablet_replica.cc:330] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae: stopping tablet replica
I20240502 02:04:18.409377 21013 raft_consensus.cc:2229] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:18.409489 21013 raft_consensus.cc:2258] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:18.410063 21013 ts_tablet_manager.cc:1875] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:18.412457 21013 ts_tablet_manager.cc:1888] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.4
I20240502 02:04:18.412526 21013 log.cc:1196] T cdae5597120c4631b6c2ffa7176d60c9 P e29339b99789441ebc37b6f327ad34ae: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/cdae5597120c4631b6c2ffa7176d60c9
I20240502 02:04:18.413141 19262 catalog_manager.cc:4830] TS e29339b99789441ebc37b6f327ad34ae (127.7.221.65:36593): tablet cdae5597120c4631b6c2ffa7176d60c9 (table rebalance_test_table_0 [id=55a2f39752324cc5843da57dfc176da0]) successfully deleted
I20240502 02:04:18.441974 19555 raft_consensus.cc:1261] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Refusing update from remote peer 6ddcf3d8a67f4601b88c7cdab5fc6fec: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 2 index: 4. (index mismatch)
I20240502 02:04:18.442580 20851 consensus_queue.cc:1035] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [LEADER]: Connected to new peer: Peer: permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:18.458636 19555 raft_consensus.cc:1261] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Refusing update from remote peer e29339b99789441ebc37b6f327ad34ae: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
I20240502 02:04:18.459030 20412 consensus_queue.cc:1035] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:18.485567 20089 consensus_queue.cc:237] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 2.4, Last appended by leader: 3, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:18.486155 20857 raft_consensus.cc:2932] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 LEADER]: Committing config change with OpId 2.5: config changed from index 3 to 5, VOTER e29339b99789441ebc37b6f327ad34ae (127.7.221.65) evicted. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } } }
I20240502 02:04:18.487380 19260 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet bc4cb3dc8c3340a88fbc50e175c68f49 with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 02:04:18.488224 19273 catalog_manager.cc:5484] T bc4cb3dc8c3340a88fbc50e175c68f49 P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 reported cstate change: config changed from index 3 to 5, VOTER e29339b99789441ebc37b6f327ad34ae (127.7.221.65) evicted. New cstate: current_term: 2 leader_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:18.490339 19397 tablet_service.cc:1514] Processing DeleteTablet for tablet bc4cb3dc8c3340a88fbc50e175c68f49 with delete_type TABLET_DATA_TOMBSTONED (TS e29339b99789441ebc37b6f327ad34ae not found in new config with opid_index 5) from {username='slave'} at 127.0.0.1:46998
I20240502 02:04:18.490581 21013 tablet_replica.cc:330] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae: stopping tablet replica
I20240502 02:04:18.490777 21013 raft_consensus.cc:2229] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:18.490890 21013 raft_consensus.cc:2258] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:18.491356 21013 ts_tablet_manager.cc:1875] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:18.492902 20223 consensus_queue.cc:237] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 2.4, Last appended by leader: 3, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } }
I20240502 02:04:18.493108 21013 ts_tablet_manager.cc:1888] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.4
I20240502 02:04:18.493175 21013 log.cc:1196] T bc4cb3dc8c3340a88fbc50e175c68f49 P e29339b99789441ebc37b6f327ad34ae: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/bc4cb3dc8c3340a88fbc50e175c68f49
I20240502 02:04:18.493453 20851 raft_consensus.cc:2932] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec [term 2 LEADER]: Committing config change with OpId 2.5: config changed from index 3 to 5, VOTER d2faf854fabf42afa2c5355009726e51 (127.7.221.66) evicted. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } } }
I20240502 02:04:18.493640 19262 catalog_manager.cc:4830] TS e29339b99789441ebc37b6f327ad34ae (127.7.221.65:36593): tablet bc4cb3dc8c3340a88fbc50e175c68f49 (table rebalance_test_table_0 [id=55a2f39752324cc5843da57dfc176da0]) successfully deleted
I20240502 02:04:18.494637 19263 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 021ef17cfdbc4537b9b3fbb49b240af8 with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 02:04:18.495414 19273 catalog_manager.cc:5484] T 021ef17cfdbc4537b9b3fbb49b240af8 P 6ddcf3d8a67f4601b88c7cdab5fc6fec reported cstate change: config changed from index 3 to 5, VOTER d2faf854fabf42afa2c5355009726e51 (127.7.221.66) evicted. New cstate: current_term: 2 leader_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "6ddcf3d8a67f4601b88c7cdab5fc6fec" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 36343 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:18.497462 19531 tablet_service.cc:1514] Processing DeleteTablet for tablet 021ef17cfdbc4537b9b3fbb49b240af8 with delete_type TABLET_DATA_TOMBSTONED (TS d2faf854fabf42afa2c5355009726e51 not found in new config with opid_index 5) from {username='slave'} at 127.0.0.1:50722
I20240502 02:04:18.497956 21084 tablet_replica.cc:330] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51: stopping tablet replica
I20240502 02:04:18.498203 21084 raft_consensus.cc:2229] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:18.498324 21084 raft_consensus.cc:2258] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:18.499262 21084 ts_tablet_manager.cc:1875] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:18.500943 21084 ts_tablet_manager.cc:1888] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.4
I20240502 02:04:18.501019 21084 log.cc:1196] T 021ef17cfdbc4537b9b3fbb49b240af8 P d2faf854fabf42afa2c5355009726e51: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/021ef17cfdbc4537b9b3fbb49b240af8
I20240502 02:04:18.501503 19261 catalog_manager.cc:4830] TS d2faf854fabf42afa2c5355009726e51 (127.7.221.66:39613): tablet 021ef17cfdbc4537b9b3fbb49b240af8 (table rebalance_test_table_3 [id=fc293509ebb744bf9e6d16de7f985b15]) successfully deleted
I20240502 02:04:18.588904 19555 raft_consensus.cc:1261] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Refusing update from remote peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 2 index: 4. (index mismatch)
I20240502 02:04:18.589434 20532 consensus_queue.cc:1035] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:18.712765 19419 consensus_queue.cc:237] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 4, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: false } }
I20240502 02:04:18.713479 20412 raft_consensus.cc:2932] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae [term 2 LEADER]: Committing config change with OpId 2.6: config changed from index 4 to 6, VOTER d2faf854fabf42afa2c5355009726e51 (127.7.221.66) evicted. New config: { opid_index: 6 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: false } } }
I20240502 02:04:18.714969 19262 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 118dc310317a4c6b8fdbeb0a710e62b0 with cas_config_opid_index 4: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 02:04:18.715986 19276 catalog_manager.cc:5484] T 118dc310317a4c6b8fdbeb0a710e62b0 P e29339b99789441ebc37b6f327ad34ae reported cstate change: config changed from index 4 to 6, VOTER d2faf854fabf42afa2c5355009726e51 (127.7.221.66) evicted. New cstate: current_term: 2 leader_uuid: "e29339b99789441ebc37b6f327ad34ae" committed_config { opid_index: 6 OBSOLETE_local: true peers { permanent_uuid: "e29339b99789441ebc37b6f327ad34ae" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 36593 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:18.718474 19531 tablet_service.cc:1514] Processing DeleteTablet for tablet 118dc310317a4c6b8fdbeb0a710e62b0 with delete_type TABLET_DATA_TOMBSTONED (TS d2faf854fabf42afa2c5355009726e51 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:50722
I20240502 02:04:18.718756 21084 tablet_replica.cc:330] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51: stopping tablet replica
I20240502 02:04:18.718932 21084 raft_consensus.cc:2229] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:18.719074 21084 raft_consensus.cc:2258] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:18.719686 21084 ts_tablet_manager.cc:1875] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:18.721683 21084 ts_tablet_manager.cc:1888] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.5
I20240502 02:04:18.721747 21084 log.cc:1196] T 118dc310317a4c6b8fdbeb0a710e62b0 P d2faf854fabf42afa2c5355009726e51: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/118dc310317a4c6b8fdbeb0a710e62b0
I20240502 02:04:18.722193 19261 catalog_manager.cc:4830] TS d2faf854fabf42afa2c5355009726e51 (127.7.221.66:39613): tablet 118dc310317a4c6b8fdbeb0a710e62b0 (table rebalance_test_table_1 [id=2653c0a691044c738cd2eea4ca609aa3]) successfully deleted
W20240502 02:04:18.820171 19260 catalog_manager.cc:5014] ChangeConfig:REMOVE_PEER RPC for tablet dc40ed8ee597423c9fc8993863b83d9e with cas_config_opid_index 4: ChangeConfig:REMOVE_PEER failed with leader 5ba05ad7c35a41f4b1a0cb4bef0d6d15 (127.7.221.67:33495) due to error UNKNOWN_ERROR; will retry: Illegal state: Leader has not yet committed an operation in its own term
I20240502 02:04:18.833357 19555 raft_consensus.cc:1261] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Refusing update from remote peer 5ba05ad7c35a41f4b1a0cb4bef0d6d15: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
I20240502 02:04:18.833809 20857 consensus_queue.cc:1035] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d2faf854fabf42afa2c5355009726e51" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 39613 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:19.126789 20089 consensus_queue.cc:237] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 2.4, Last appended by leader: 3, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:19.127444 20857 raft_consensus.cc:2932] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 LEADER]: Committing config change with OpId 2.5: config changed from index 3 to 5, VOTER d2faf854fabf42afa2c5355009726e51 (127.7.221.66) evicted. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } } }
I20240502 02:04:19.128680 19260 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d77eb55afc0e4dd796390ebb3961d0fc with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 7)
I20240502 02:04:19.129621 19273 catalog_manager.cc:5484] T d77eb55afc0e4dd796390ebb3961d0fc P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 reported cstate change: config changed from index 3 to 5, VOTER d2faf854fabf42afa2c5355009726e51 (127.7.221.66) evicted. New cstate: current_term: 2 leader_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:19.131898 19531 tablet_service.cc:1514] Processing DeleteTablet for tablet d77eb55afc0e4dd796390ebb3961d0fc with delete_type TABLET_DATA_TOMBSTONED (TS d2faf854fabf42afa2c5355009726e51 not found in new config with opid_index 5) from {username='slave'} at 127.0.0.1:50722
I20240502 02:04:19.132158 21084 tablet_replica.cc:330] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51: stopping tablet replica
I20240502 02:04:19.132299 21084 raft_consensus.cc:2229] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:19.132407 21084 raft_consensus.cc:2258] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:19.132915 21084 ts_tablet_manager.cc:1875] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:19.134562 21084 ts_tablet_manager.cc:1888] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.4
I20240502 02:04:19.134668 21084 log.cc:1196] T d77eb55afc0e4dd796390ebb3961d0fc P d2faf854fabf42afa2c5355009726e51: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/d77eb55afc0e4dd796390ebb3961d0fc
I20240502 02:04:19.135146 19261 catalog_manager.cc:4830] TS d2faf854fabf42afa2c5355009726e51 (127.7.221.66:39613): tablet d77eb55afc0e4dd796390ebb3961d0fc (table rebalance_test_table_1 [id=2653c0a691044c738cd2eea4ca609aa3]) successfully deleted
I20240502 02:04:19.355029 20089 consensus_queue.cc:237] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 4, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: true peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } }
I20240502 02:04:19.355710 20857 raft_consensus.cc:2932] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 [term 2 LEADER]: Committing config change with OpId 2.6: config changed from index 4 to 6, VOTER d2faf854fabf42afa2c5355009726e51 (127.7.221.66) evicted. New config: { opid_index: 6 OBSOLETE_local: true peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } } }
I20240502 02:04:19.357023 19260 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet dc40ed8ee597423c9fc8993863b83d9e with cas_config_opid_index 4: ChangeConfig:REMOVE_PEER succeeded (attempt 7)
I20240502 02:04:19.357946 19273 catalog_manager.cc:5484] T dc40ed8ee597423c9fc8993863b83d9e P 5ba05ad7c35a41f4b1a0cb4bef0d6d15 reported cstate change: config changed from index 4 to 6, VOTER d2faf854fabf42afa2c5355009726e51 (127.7.221.66) evicted. New cstate: current_term: 2 leader_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" committed_config { opid_index: 6 OBSOLETE_local: true peers { permanent_uuid: "5ba05ad7c35a41f4b1a0cb4bef0d6d15" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 33495 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:19.360077 19531 tablet_service.cc:1514] Processing DeleteTablet for tablet dc40ed8ee597423c9fc8993863b83d9e with delete_type TABLET_DATA_TOMBSTONED (TS d2faf854fabf42afa2c5355009726e51 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:50722
I20240502 02:04:19.360316 21084 tablet_replica.cc:330] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51: stopping tablet replica
I20240502 02:04:19.360453 21084 raft_consensus.cc:2229] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:19.360556 21084 raft_consensus.cc:2258] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:19.361053 21084 ts_tablet_manager.cc:1875] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:19.362700 21084 ts_tablet_manager.cc:1888] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.5
I20240502 02:04:19.362758 21084 log.cc:1196] T dc40ed8ee597423c9fc8993863b83d9e P d2faf854fabf42afa2c5355009726e51: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/dc40ed8ee597423c9fc8993863b83d9e
I20240502 02:04:19.363193 19261 catalog_manager.cc:4830] TS d2faf854fabf42afa2c5355009726e51 (127.7.221.66:39613): tablet dc40ed8ee597423c9fc8993863b83d9e (table rebalance_test_table_3 [id=fc293509ebb744bf9e6d16de7f985b15]) successfully deleted
I20240502 02:04:19.435216 20070 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 02:04:19.436286 19531 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:19.437098 20202 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:19.443635 19397 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:19.525475 20070 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 02:04:19.527377 19531 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:19.529035 20202 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:19.533246 19397 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:19.617661 20202 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:19.617666 20070 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 02:04:19.618346 19397 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:19.620048 19531 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
47aad5d269304635b35d456d7e868c6b | 127.7.221.126:43599 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.7.221.84:36725 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+---------+----------+----------------+-----------------
5ba05ad7c35a41f4b1a0cb4bef0d6d15 | 127.7.221.67:33495 | HEALTHY | <none> | 4 | 0
6ddcf3d8a67f4601b88c7cdab5fc6fec | 127.7.221.68:36343 | HEALTHY | <none> | 3 | 0
d2faf854fabf42afa2c5355009726e51 | 127.7.221.66:39613 | HEALTHY | <none> | 3 | 0
e29339b99789441ebc37b6f327ad34ae | 127.7.221.65:36593 | HEALTHY | <none> | 3 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 4
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 4 server(s) checked
local_ip_for_outbound_sockets | 127.7.221.65 | experimental | 127.7.221.65:36593
local_ip_for_outbound_sockets | 127.7.221.66 | experimental | 127.7.221.66:39613
local_ip_for_outbound_sockets | 127.7.221.67 | experimental | 127.7.221.67:33495
local_ip_for_outbound_sockets | 127.7.221.68 | experimental | 127.7.221.68:36343
never_fsync | true | unsafe,advanced | all 4 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 4 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 4 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 4 server(s) checked
server_dump_info_format | pb | hidden | all 4 server(s) checked
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.7.221.65:36593
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.7.221.66:39613
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.7.221.67:33495
server_dump_info_path | /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_1.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.7.221.68:36343
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.7.221.84:36725 | all 4 server(s) checked
time_source | builtin | all 4 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 5 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
TestTable | 1 | HEALTHY | 1 | 1 | 0 | 0 | 0
rebalance_test_table_0 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 4
Maximum | 4
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 4
Tables | 5
Tablets | 13
Replicas | 13
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240502 02:04:19.651613 8053 log_verifier.cc:126] Checking tablet 021ef17cfdbc4537b9b3fbb49b240af8
I20240502 02:04:19.659941 8053 log_verifier.cc:177] Verified matching terms for 5 ops in tablet 021ef17cfdbc4537b9b3fbb49b240af8
I20240502 02:04:19.660033 8053 log_verifier.cc:126] Checking tablet 118dc310317a4c6b8fdbeb0a710e62b0
I20240502 02:04:19.667502 8053 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 118dc310317a4c6b8fdbeb0a710e62b0
I20240502 02:04:19.667567 8053 log_verifier.cc:126] Checking tablet 15436057adfe47679cbe78552d8c29b0
I20240502 02:04:19.675050 8053 log_verifier.cc:177] Verified matching terms for 2 ops in tablet 15436057adfe47679cbe78552d8c29b0
I20240502 02:04:19.675109 8053 log_verifier.cc:126] Checking tablet 61234e24e53049f5be6c60dbbd170328
I20240502 02:04:19.682366 8053 log_verifier.cc:177] Verified matching terms for 5 ops in tablet 61234e24e53049f5be6c60dbbd170328
I20240502 02:04:19.682433 8053 log_verifier.cc:126] Checking tablet 8bcff5fdda3c49c992c2cb78014f4673
I20240502 02:04:19.689779 8053 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 8bcff5fdda3c49c992c2cb78014f4673
I20240502 02:04:19.689838 8053 log_verifier.cc:126] Checking tablet bc4cb3dc8c3340a88fbc50e175c68f49
I20240502 02:04:19.697393 8053 log_verifier.cc:177] Verified matching terms for 5 ops in tablet bc4cb3dc8c3340a88fbc50e175c68f49
I20240502 02:04:19.697458 8053 log_verifier.cc:126] Checking tablet c07aa9911ae24eb7b8221e5245ad56be
I20240502 02:04:19.704535 8053 log_verifier.cc:177] Verified matching terms for 1 ops in tablet c07aa9911ae24eb7b8221e5245ad56be
I20240502 02:04:19.704600 8053 log_verifier.cc:126] Checking tablet c7aa1292397343e69abf05758d4eca7f
I20240502 02:04:19.711898 8053 log_verifier.cc:177] Verified matching terms for 5 ops in tablet c7aa1292397343e69abf05758d4eca7f
I20240502 02:04:19.711975 8053 log_verifier.cc:126] Checking tablet cdae5597120c4631b6c2ffa7176d60c9
I20240502 02:04:19.719494 8053 log_verifier.cc:177] Verified matching terms for 5 ops in tablet cdae5597120c4631b6c2ffa7176d60c9
I20240502 02:04:19.719563 8053 log_verifier.cc:126] Checking tablet d77eb55afc0e4dd796390ebb3961d0fc
I20240502 02:04:19.727154 8053 log_verifier.cc:177] Verified matching terms for 5 ops in tablet d77eb55afc0e4dd796390ebb3961d0fc
I20240502 02:04:19.727221 8053 log_verifier.cc:126] Checking tablet d9243491bd8142339827c5b7718cdfd1
I20240502 02:04:19.734771 8053 log_verifier.cc:177] Verified matching terms for 2 ops in tablet d9243491bd8142339827c5b7718cdfd1
I20240502 02:04:19.734828 8053 log_verifier.cc:126] Checking tablet dc40ed8ee597423c9fc8993863b83d9e
I20240502 02:04:19.742655 8053 log_verifier.cc:177] Verified matching terms for 6 ops in tablet dc40ed8ee597423c9fc8993863b83d9e
I20240502 02:04:19.742717 8053 log_verifier.cc:126] Checking tablet f788ee8d1d2043c8b7860111eafffba8
I20240502 02:04:19.750053 8053 log_verifier.cc:177] Verified matching terms for 1 ops in tablet f788ee8d1d2043c8b7860111eafffba8
I20240502 02:04:19.750535 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 19337
I20240502 02:04:19.758023 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 19470
I20240502 02:04:19.764209 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 20007
I20240502 02:04:19.770345 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 20140
I20240502 02:04:19.776399 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 19244
2024-05-02T02:04:19Z chronyd exiting
[ OK ] RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove/1 (10240 ms)
[----------] 1 test from RebalancerAndSingleReplicaTablets (10240 ms total)
[----------] 2 tests from LocationAwareRebalancingParamTest
[ RUN ] LocationAwareRebalancingParamTest.Rebalance/idx3_rf3_t3_l_3_3
I20240502 02:04:19.810417 8053 test_util.cc:268] Using random seed: 90737772
I20240502 02:04:19.811249 8053 ts_itest-base.cc:112] Starting cluster with:
I20240502 02:04:19.811287 8053 ts_itest-base.cc:113] --------------
I20240502 02:04:19.811304 8053 ts_itest-base.cc:114] 6 tablet servers
I20240502 02:04:19.811319 8053 ts_itest-base.cc:115] 3 replicas per TS
I20240502 02:04:19.811336 8053 ts_itest-base.cc:116] --------------
2024-05-02T02:04:19Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T02:04:19Z Disabled control of system clock
I20240502 02:04:19.819980 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.7.221.126:38635
--webserver_interface=127.7.221.126
--webserver_port=0
--builtin_ntp_servers=127.7.221.84:44609
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.7.221.126:38635
--location_mapping_cmd=/tmp/dist-test-tasko3zIVx/build/debug/bin/testdata/assign-location.py --state_store=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/location-assignment.state --map /A:3 --map /B:3
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false
--master_client_location_assignment_enabled=false
--allow_unsafe_replication_factor with env {}
W20240502 02:04:19.918731 21265 flags.cc:425] Enabled unsafe flag: --allow_unsafe_replication_factor=true
W20240502 02:04:19.918959 21265 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:19.918983 21265 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:19.919020 21265 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:19.921792 21265 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:19.921850 21265 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 02:04:19.921885 21265 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:19.921916 21265 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 02:04:19.921931 21265 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 02:04:19.925199 21265 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:44609
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
--allow_unsafe_replication_factor=true
--enable_range_replica_placement=false
--location_mapping_cmd=/tmp/dist-test-tasko3zIVx/build/debug/bin/testdata/assign-location.py --state_store=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/location-assignment.state --map /A:3 --map /B:3
--ipki_ca_key_size=768
--master_addresses=127.7.221.126:38635
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.7.221.126:38635
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.7.221.126
--webserver_port=0
--never_fsync=true
--heap_profile_path=/tmp/kudu.21265
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:19.925992 21265 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:19.927012 21265 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:19.933254 21271 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:19.933943 21273 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:19.934572 21270 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:19.935232 21265 server_base.cc:1034] running on GCE node
I20240502 02:04:19.935739 21265 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:19.936625 21265 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:19.937793 21265 hybrid_clock.cc:648] HybridClock initialized: now 1714615459937771 us; error 32 us; skew 500 ppm
I20240502 02:04:19.939523 21265 webserver.cc:460] Webserver started at http://127.7.221.126:37029/ using document root <none> and password file <none>
I20240502 02:04:19.940025 21265 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:19.940093 21265 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:19.940248 21265 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:19.941751 21265 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "12ef0c9785c84ae394ad78b1bb3efb80"
format_stamp: "Formatted at 2024-05-02 02:04:19 on dist-test-slave-dmxt"
I20240502 02:04:19.942221 21265 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "12ef0c9785c84ae394ad78b1bb3efb80"
format_stamp: "Formatted at 2024-05-02 02:04:19 on dist-test-slave-dmxt"
I20240502 02:04:19.945391 21265 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.004s sys 0.001s
I20240502 02:04:19.947496 21279 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:19.948372 21265 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:19.948503 21265 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
uuid: "12ef0c9785c84ae394ad78b1bb3efb80"
format_stamp: "Formatted at 2024-05-02 02:04:19 on dist-test-slave-dmxt"
I20240502 02:04:19.948583 21265 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:19.965577 21265 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:19.966199 21265 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:19.966353 21265 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:19.972235 21265 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.126:38635
I20240502 02:04:19.972240 21331 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.126:38635 every 8 connection(s)
I20240502 02:04:19.973138 21265 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 02:04:19.974551 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 21265
I20240502 02:04:19.974721 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 02:04:19.975703 21332 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:19.980755 21332 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80: Bootstrap starting.
I20240502 02:04:19.982833 21332 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:19.983636 21332 log.cc:824] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:19.985137 21332 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80: No bootstrap required, opened a new log
I20240502 02:04:19.987435 21332 raft_consensus.cc:348] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12ef0c9785c84ae394ad78b1bb3efb80" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 38635 } }
I20240502 02:04:19.987589 21332 raft_consensus.cc:374] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:19.987640 21332 raft_consensus.cc:729] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 12ef0c9785c84ae394ad78b1bb3efb80, State: Initialized, Role: FOLLOWER
I20240502 02:04:19.987978 21332 consensus_queue.cc:260] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12ef0c9785c84ae394ad78b1bb3efb80" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 38635 } }
I20240502 02:04:19.988083 21332 raft_consensus.cc:388] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:19.988123 21332 raft_consensus.cc:482] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:19.988191 21332 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:19.988813 21332 raft_consensus.cc:504] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12ef0c9785c84ae394ad78b1bb3efb80" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 38635 } }
I20240502 02:04:19.989099 21332 leader_election.cc:302] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 12ef0c9785c84ae394ad78b1bb3efb80; no voters:
I20240502 02:04:19.989393 21332 leader_election.cc:288] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:19.989737 21337 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:19.989933 21337 raft_consensus.cc:686] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [term 1 LEADER]: Becoming Leader. State: Replica: 12ef0c9785c84ae394ad78b1bb3efb80, State: Running, Role: LEADER
I20240502 02:04:19.990185 21337 consensus_queue.cc:237] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12ef0c9785c84ae394ad78b1bb3efb80" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 38635 } }
I20240502 02:04:19.990335 21332 sys_catalog.cc:564] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [sys.catalog]: configured and running, proceeding with master startup.
I20240502 02:04:19.991425 21338 sys_catalog.cc:455] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "12ef0c9785c84ae394ad78b1bb3efb80" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12ef0c9785c84ae394ad78b1bb3efb80" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 38635 } } }
I20240502 02:04:19.991525 21338 sys_catalog.cc:458] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [sys.catalog]: This master's current role is: LEADER
I20240502 02:04:19.991812 21339 sys_catalog.cc:455] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 12ef0c9785c84ae394ad78b1bb3efb80. Latest consensus state: current_term: 1 leader_uuid: "12ef0c9785c84ae394ad78b1bb3efb80" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12ef0c9785c84ae394ad78b1bb3efb80" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 38635 } } }
I20240502 02:04:19.991891 21339 sys_catalog.cc:458] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80 [sys.catalog]: This master's current role is: LEADER
I20240502 02:04:19.991986 21347 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 02:04:19.994421 21347 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 02:04:19.998661 21347 catalog_manager.cc:1347] Generated new cluster ID: b7d8b954a08041e28b18ec18025279ba
I20240502 02:04:19.998715 21347 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 02:04:20.009954 21347 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 02:04:20.010792 21347 catalog_manager.cc:1504] Loading token signing keys...
I20240502 02:04:20.016934 21347 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 12ef0c9785c84ae394ad78b1bb3efb80: Generated new TSK 0
I20240502 02:04:20.017501 21347 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 02:04:20.019894 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.65:0
--local_ip_for_outbound_sockets=127.7.221.65
--webserver_interface=127.7.221.65
--webserver_port=0
--tserver_master_addrs=127.7.221.126:38635
--builtin_ntp_servers=127.7.221.84:44609
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 02:04:20.121567 21357 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:20.121781 21357 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:20.121824 21357 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:20.124799 21357 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:20.124856 21357 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:20.124945 21357 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.65
I20240502 02:04:20.128405 21357 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:44609
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.7.221.65
--webserver_port=0
--tserver_master_addrs=127.7.221.126:38635
--never_fsync=true
--heap_profile_path=/tmp/kudu.21357
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.65
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:20.129246 21357 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:20.130314 21357 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:20.137940 21363 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:20.138531 21362 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:20.139125 21365 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:20.139904 21357 server_base.cc:1034] running on GCE node
I20240502 02:04:20.140286 21357 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:20.140974 21357 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:20.142154 21357 hybrid_clock.cc:648] HybridClock initialized: now 1714615460142141 us; error 25 us; skew 500 ppm
I20240502 02:04:20.144155 21357 webserver.cc:460] Webserver started at http://127.7.221.65:42709/ using document root <none> and password file <none>
I20240502 02:04:20.144694 21357 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:20.144747 21357 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:20.144913 21357 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:20.146500 21357 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "e72358c248634f30be5f9becf0ec0fa0"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:20.146991 21357 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "e72358c248634f30be5f9becf0ec0fa0"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:20.150220 21357 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 02:04:20.152307 21371 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:20.153247 21357 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:20.153352 21357 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "e72358c248634f30be5f9becf0ec0fa0"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:20.153430 21357 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:20.179116 21357 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:20.179751 21357 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:20.179924 21357 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:20.180462 21357 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:20.181365 21357 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:20.181416 21357 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:20.181474 21357 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:20.181509 21357 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:20.189968 21357 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.65:38519
I20240502 02:04:20.189996 21485 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.65:38519 every 8 connection(s)
I20240502 02:04:20.190979 21357 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 02:04:20.195405 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 21357
I20240502 02:04:20.195591 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 02:04:20.197734 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.66:0
--local_ip_for_outbound_sockets=127.7.221.66
--webserver_interface=127.7.221.66
--webserver_port=0
--tserver_master_addrs=127.7.221.126:38635
--builtin_ntp_servers=127.7.221.84:44609
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 02:04:20.200047 21486 heartbeater.cc:347] Connected to a master server at 127.7.221.126:38635
I20240502 02:04:20.200250 21486 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:20.200724 21486 heartbeater.cc:511] Master 127.7.221.126:38635 requested a full tablet report, sending...
I20240502 02:04:20.237201 21296 ts_manager.cc:194] Registered new tserver with Master: e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519)
I20240502 02:04:20.238539 21296 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.65:35451
W20240502 02:04:20.300303 21489 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:20.300487 21489 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:20.300525 21489 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:20.303400 21489 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:20.303452 21489 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:20.303526 21489 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.66
I20240502 02:04:20.306934 21489 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:44609
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.7.221.66
--webserver_port=0
--tserver_master_addrs=127.7.221.126:38635
--never_fsync=true
--heap_profile_path=/tmp/kudu.21489
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.66
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:20.307675 21489 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:20.308624 21489 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:20.315824 21498 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:20.316478 21495 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:20.317051 21496 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:20.317787 21489 server_base.cc:1034] running on GCE node
I20240502 02:04:20.318118 21489 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:20.318645 21489 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:20.319788 21489 hybrid_clock.cc:648] HybridClock initialized: now 1714615460319779 us; error 20 us; skew 500 ppm
I20240502 02:04:20.321334 21489 webserver.cc:460] Webserver started at http://127.7.221.66:34901/ using document root <none> and password file <none>
I20240502 02:04:20.321831 21489 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:20.321903 21489 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:20.322088 21489 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:20.323601 21489 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "252575ab96ac4a079bab6ade63792782"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:20.324012 21489 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "252575ab96ac4a079bab6ade63792782"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:20.327061 21489 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.001s
I20240502 02:04:20.328884 21504 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:20.329741 21489 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:20.329838 21489 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "252575ab96ac4a079bab6ade63792782"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:20.329916 21489 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:20.348191 21489 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:20.348735 21489 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:20.348886 21489 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:20.349365 21489 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:20.350215 21489 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:20.350287 21489 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:20.350343 21489 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:20.350369 21489 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:20.358366 21489 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.66:34219
I20240502 02:04:20.358417 21620 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.66:34219 every 8 connection(s)
I20240502 02:04:20.359263 21489 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 02:04:20.363584 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 21489
I20240502 02:04:20.363718 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 02:04:20.365859 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.67:0
--local_ip_for_outbound_sockets=127.7.221.67
--webserver_interface=127.7.221.67
--webserver_port=0
--tserver_master_addrs=127.7.221.126:38635
--builtin_ntp_servers=127.7.221.84:44609
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 02:04:20.367796 21621 heartbeater.cc:347] Connected to a master server at 127.7.221.126:38635
I20240502 02:04:20.368031 21621 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:20.368585 21621 heartbeater.cc:511] Master 127.7.221.126:38635 requested a full tablet report, sending...
I20240502 02:04:20.401988 21296 ts_manager.cc:194] Registered new tserver with Master: 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219)
I20240502 02:04:20.402829 21296 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.66:48463
W20240502 02:04:20.470512 21624 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:20.470757 21624 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:20.470813 21624 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:20.473523 21624 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:20.473582 21624 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:20.473642 21624 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.67
I20240502 02:04:20.476997 21624 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:44609
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.7.221.67
--webserver_port=0
--tserver_master_addrs=127.7.221.126:38635
--never_fsync=true
--heap_profile_path=/tmp/kudu.21624
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.67
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:20.477881 21624 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:20.479005 21624 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:20.486541 21630 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:20.486650 21633 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:20.487267 21631 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:20.487946 21624 server_base.cc:1034] running on GCE node
I20240502 02:04:20.488291 21624 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:20.488806 21624 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:20.489953 21624 hybrid_clock.cc:648] HybridClock initialized: now 1714615460489941 us; error 22 us; skew 500 ppm
I20240502 02:04:20.491634 21624 webserver.cc:460] Webserver started at http://127.7.221.67:41919/ using document root <none> and password file <none>
I20240502 02:04:20.492249 21624 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:20.492317 21624 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:20.492498 21624 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:20.494108 21624 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "0983e154926c44ceb508483878dba5a0"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:20.494560 21624 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "0983e154926c44ceb508483878dba5a0"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:20.497797 21624 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.000s
I20240502 02:04:20.499788 21639 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:20.500685 21624 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:20.500790 21624 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "0983e154926c44ceb508483878dba5a0"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:20.500877 21624 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:20.514835 21624 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:20.515367 21624 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:20.515529 21624 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:20.516029 21624 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:20.516950 21624 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:20.517000 21624 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:20.517078 21624 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:20.517102 21624 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:20.524993 21624 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.67:35557
I20240502 02:04:20.525028 21754 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.67:35557 every 8 connection(s)
I20240502 02:04:20.525919 21624 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 02:04:20.532088 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 21624
I20240502 02:04:20.532232 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 02:04:20.534432 21755 heartbeater.cc:347] Connected to a master server at 127.7.221.126:38635
I20240502 02:04:20.534693 21755 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:20.534826 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.68:0
--local_ip_for_outbound_sockets=127.7.221.68
--webserver_interface=127.7.221.68
--webserver_port=0
--tserver_master_addrs=127.7.221.126:38635
--builtin_ntp_servers=127.7.221.84:44609
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 02:04:20.535301 21755 heartbeater.cc:511] Master 127.7.221.126:38635 requested a full tablet report, sending...
I20240502 02:04:20.567974 21296 ts_manager.cc:194] Registered new tserver with Master: 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:20.568878 21296 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.67:41269
W20240502 02:04:20.639827 21758 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:20.640044 21758 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:20.640084 21758 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:20.642946 21758 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:20.643016 21758 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:20.643085 21758 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.68
I20240502 02:04:20.646719 21758 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:44609
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.68:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.7.221.68
--webserver_port=0
--tserver_master_addrs=127.7.221.126:38635
--never_fsync=true
--heap_profile_path=/tmp/kudu.21758
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.68
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:20.647651 21758 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:20.648795 21758 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:20.657286 21764 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:20.657375 21765 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:20.657951 21767 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:20.658687 21758 server_base.cc:1034] running on GCE node
I20240502 02:04:20.659094 21758 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:20.659761 21758 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:20.660909 21758 hybrid_clock.cc:648] HybridClock initialized: now 1714615460660885 us; error 34 us; skew 500 ppm
I20240502 02:04:20.662755 21758 webserver.cc:460] Webserver started at http://127.7.221.68:42765/ using document root <none> and password file <none>
I20240502 02:04:20.663329 21758 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:20.663394 21758 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:20.663570 21758 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:20.665261 21758 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "d590702730234f799e2a26de695bd7a2"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:20.665741 21758 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "d590702730234f799e2a26de695bd7a2"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:20.669102 21758 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.004s
I20240502 02:04:20.671181 21773 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:20.672144 21758 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:20.672262 21758 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "d590702730234f799e2a26de695bd7a2"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:20.672341 21758 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:20.685585 21758 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:20.686152 21758 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:20.686301 21758 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:20.686867 21758 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:20.687672 21758 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:20.687718 21758 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:20.687774 21758 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:20.687798 21758 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:20.695892 21758 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.68:46455
I20240502 02:04:20.695922 21889 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.68:46455 every 8 connection(s)
I20240502 02:04:20.696822 21758 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 02:04:20.701174 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 21758
I20240502 02:04:20.701303 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 02:04:20.703459 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.69:0
--local_ip_for_outbound_sockets=127.7.221.69
--webserver_interface=127.7.221.69
--webserver_port=0
--tserver_master_addrs=127.7.221.126:38635
--builtin_ntp_servers=127.7.221.84:44609
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 02:04:20.705703 21890 heartbeater.cc:347] Connected to a master server at 127.7.221.126:38635
I20240502 02:04:20.705927 21890 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:20.706434 21890 heartbeater.cc:511] Master 127.7.221.126:38635 requested a full tablet report, sending...
I20240502 02:04:20.740097 21296 ts_manager.cc:194] Registered new tserver with Master: d590702730234f799e2a26de695bd7a2 (127.7.221.68:46455)
I20240502 02:04:20.741029 21296 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.68:43975
W20240502 02:04:20.807262 21893 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:20.807461 21893 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:20.807502 21893 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:20.810122 21893 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:20.810178 21893 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:20.810233 21893 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.69
I20240502 02:04:20.813454 21893 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:44609
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.69:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.7.221.69
--webserver_port=0
--tserver_master_addrs=127.7.221.126:38635
--never_fsync=true
--heap_profile_path=/tmp/kudu.21893
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.69
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:20.814224 21893 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:20.815244 21893 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:20.823037 21899 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:20.823308 21900 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:20.823729 21902 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:20.824316 21893 server_base.cc:1034] running on GCE node
I20240502 02:04:20.824666 21893 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:20.825227 21893 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:20.826391 21893 hybrid_clock.cc:648] HybridClock initialized: now 1714615460826372 us; error 31 us; skew 500 ppm
I20240502 02:04:20.828066 21893 webserver.cc:460] Webserver started at http://127.7.221.69:44923/ using document root <none> and password file <none>
I20240502 02:04:20.828590 21893 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:20.828649 21893 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:20.828814 21893 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:20.830396 21893 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "5e9df19de7264d1a9ee930ee770a8701"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:20.830842 21893 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "5e9df19de7264d1a9ee930ee770a8701"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:20.833863 21893 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.000s
I20240502 02:04:20.835860 21908 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:20.836798 21893 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:20.836897 21893 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "5e9df19de7264d1a9ee930ee770a8701"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:20.836970 21893 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:20.855559 21893 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:20.856163 21893 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:20.856323 21893 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:20.856834 21893 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:20.857755 21893 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:20.857800 21893 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:20.857836 21893 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:20.857853 21893 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:20.865422 21893 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.69:45619
I20240502 02:04:20.865449 22023 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.69:45619 every 8 connection(s)
I20240502 02:04:20.866335 21893 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 02:04:20.869845 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 21893
I20240502 02:04:20.869952 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 02:04:20.872085 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.70:0
--local_ip_for_outbound_sockets=127.7.221.70
--webserver_interface=127.7.221.70
--webserver_port=0
--tserver_master_addrs=127.7.221.126:38635
--builtin_ntp_servers=127.7.221.84:44609
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 02:04:20.874657 22024 heartbeater.cc:347] Connected to a master server at 127.7.221.126:38635
I20240502 02:04:20.874871 22024 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:20.875476 22024 heartbeater.cc:511] Master 127.7.221.126:38635 requested a full tablet report, sending...
I20240502 02:04:20.923040 21296 ts_manager.cc:194] Registered new tserver with Master: 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69:45619)
I20240502 02:04:20.923766 21296 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.69:43137
W20240502 02:04:20.990609 22028 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:20.990839 22028 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:20.990896 22028 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:20.993695 22028 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:20.993759 22028 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:20.993867 22028 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.70
I20240502 02:04:20.997328 22028 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:44609
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.70:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.7.221.70
--webserver_port=0
--tserver_master_addrs=127.7.221.126:38635
--never_fsync=true
--heap_profile_path=/tmp/kudu.22028
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.70
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:20.998167 22028 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:20.999233 22028 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:21.007018 22035 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:21.007578 22034 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:21.008247 22037 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:21.008785 22028 server_base.cc:1034] running on GCE node
I20240502 02:04:21.009150 22028 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:21.009716 22028 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:21.010855 22028 hybrid_clock.cc:648] HybridClock initialized: now 1714615461010840 us; error 32 us; skew 500 ppm
I20240502 02:04:21.012583 22028 webserver.cc:460] Webserver started at http://127.7.221.70:37405/ using document root <none> and password file <none>
I20240502 02:04:21.013126 22028 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:21.013183 22028 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:21.013362 22028 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:21.015097 22028 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/instance:
uuid: "2590825fe67b48c9adb5894c322237dc"
format_stamp: "Formatted at 2024-05-02 02:04:21 on dist-test-slave-dmxt"
I20240502 02:04:21.015559 22028 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal/instance:
uuid: "2590825fe67b48c9adb5894c322237dc"
format_stamp: "Formatted at 2024-05-02 02:04:21 on dist-test-slave-dmxt"
I20240502 02:04:21.018842 22028 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 02:04:21.020867 22043 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:21.021824 22028 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:21.021930 22028 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "2590825fe67b48c9adb5894c322237dc"
format_stamp: "Formatted at 2024-05-02 02:04:21 on dist-test-slave-dmxt"
I20240502 02:04:21.022010 22028 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:21.044433 22028 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:21.045091 22028 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:21.045260 22028 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:21.045820 22028 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:21.046757 22028 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:21.046805 22028 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:21.046865 22028 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:21.046892 22028 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:21.054836 22028 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.70:39691
I20240502 02:04:21.054885 22158 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.70:39691 every 8 connection(s)
I20240502 02:04:21.055754 22028 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 02:04:21.058243 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 22028
I20240502 02:04:21.058364 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240502 02:04:21.064412 22159 heartbeater.cc:347] Connected to a master server at 127.7.221.126:38635
I20240502 02:04:21.064654 22159 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:21.065162 22159 heartbeater.cc:511] Master 127.7.221.126:38635 requested a full tablet report, sending...
I20240502 02:04:21.098021 21296 ts_manager.cc:194] Registered new tserver with Master: 2590825fe67b48c9adb5894c322237dc (127.7.221.70:39691)
I20240502 02:04:21.098677 21296 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.70:36775
I20240502 02:04:21.109439 8053 external_mini_cluster.cc:930] 6 TS(s) registered with all masters
I20240502 02:04:21.122030 8053 external_mini_cluster.cc:930] 6 TS(s) registered with all masters
I20240502 02:04:21.122117 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 21893
I20240502 02:04:21.131659 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 22028
I20240502 02:04:21.138721 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 21758
I20240502 02:04:21.241114 21486 heartbeater.cc:503] Master 127.7.221.126:38635 was elected leader, sending a full tablet report...
I20240502 02:04:21.405392 21621 heartbeater.cc:503] Master 127.7.221.126:38635 was elected leader, sending a full tablet report...
I20240502 02:04:21.571113 21755 heartbeater.cc:503] Master 127.7.221.126:38635 was elected leader, sending a full tablet report...
I20240502 02:04:24.895614 21296 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51734:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 02:04:24.898736 21296 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_0 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20240502 02:04:24.915038 21688 tablet_service.cc:1467] Processing CreateTablet for tablet 1683e87385ab4050824a71508d3c3f5b (DEFAULT_TABLE table=rebalance_test_table_0 [id=e1df1cf914834ab4bf5ac68bc944cbcc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:24.915086 21550 tablet_service.cc:1467] Processing CreateTablet for tablet 2fd616a594f245a3b13353f1c9cb9064 (DEFAULT_TABLE table=rebalance_test_table_0 [id=e1df1cf914834ab4bf5ac68bc944cbcc]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:24.915647 21686 tablet_service.cc:1467] Processing CreateTablet for tablet 2fd616a594f245a3b13353f1c9cb9064 (DEFAULT_TABLE table=rebalance_test_table_0 [id=e1df1cf914834ab4bf5ac68bc944cbcc]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:24.915935 21550 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2fd616a594f245a3b13353f1c9cb9064. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:24.915942 21688 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1683e87385ab4050824a71508d3c3f5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:24.915038 21687 tablet_service.cc:1467] Processing CreateTablet for tablet 15a6b979fd044fe9b58f86ffe137b604 (DEFAULT_TABLE table=rebalance_test_table_0 [id=e1df1cf914834ab4bf5ac68bc944cbcc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:24.916237 21687 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 15a6b979fd044fe9b58f86ffe137b604. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:24.916401 21686 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2fd616a594f245a3b13353f1c9cb9064. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:24.915037 21552 tablet_service.cc:1467] Processing CreateTablet for tablet 1683e87385ab4050824a71508d3c3f5b (DEFAULT_TABLE table=rebalance_test_table_0 [id=e1df1cf914834ab4bf5ac68bc944cbcc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:24.918908 21552 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1683e87385ab4050824a71508d3c3f5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:24.921859 22178 tablet_bootstrap.cc:492] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0: Bootstrap starting.
I20240502 02:04:24.915037 21551 tablet_service.cc:1467] Processing CreateTablet for tablet 15a6b979fd044fe9b58f86ffe137b604 (DEFAULT_TABLE table=rebalance_test_table_0 [id=e1df1cf914834ab4bf5ac68bc944cbcc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:24.922466 21551 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 15a6b979fd044fe9b58f86ffe137b604. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:24.923045 22179 tablet_bootstrap.cc:492] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782: Bootstrap starting.
I20240502 02:04:24.924141 22178 tablet_bootstrap.cc:654] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:24.924841 22179 tablet_bootstrap.cc:654] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:24.926085 22179 log.cc:824] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:24.927389 22178 log.cc:824] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:24.927771 21418 tablet_service.cc:1467] Processing CreateTablet for tablet 15a6b979fd044fe9b58f86ffe137b604 (DEFAULT_TABLE table=rebalance_test_table_0 [id=e1df1cf914834ab4bf5ac68bc944cbcc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:24.928669 21418 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 15a6b979fd044fe9b58f86ffe137b604. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:24.929347 22178 tablet_bootstrap.cc:492] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0: No bootstrap required, opened a new log
I20240502 02:04:24.929510 22178 ts_tablet_manager.cc:1367] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0: Time spent bootstrapping tablet: real 0.008s user 0.004s sys 0.000s
I20240502 02:04:24.927771 21417 tablet_service.cc:1467] Processing CreateTablet for tablet 2fd616a594f245a3b13353f1c9cb9064 (DEFAULT_TABLE table=rebalance_test_table_0 [id=e1df1cf914834ab4bf5ac68bc944cbcc]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:24.930853 21417 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2fd616a594f245a3b13353f1c9cb9064. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:24.931777 22178 raft_consensus.cc:348] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } }
I20240502 02:04:24.931946 22178 raft_consensus.cc:374] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:24.932016 22178 raft_consensus.cc:729] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0983e154926c44ceb508483878dba5a0, State: Initialized, Role: FOLLOWER
I20240502 02:04:24.927909 21419 tablet_service.cc:1467] Processing CreateTablet for tablet 1683e87385ab4050824a71508d3c3f5b (DEFAULT_TABLE table=rebalance_test_table_0 [id=e1df1cf914834ab4bf5ac68bc944cbcc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:24.932390 22178 consensus_queue.cc:260] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } }
I20240502 02:04:24.932601 21419 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1683e87385ab4050824a71508d3c3f5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:24.933142 22178 ts_tablet_manager.cc:1398] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0: Time spent starting tablet: real 0.004s user 0.004s sys 0.000s
I20240502 02:04:24.933422 22178 tablet_bootstrap.cc:492] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0: Bootstrap starting.
I20240502 02:04:24.934243 22178 tablet_bootstrap.cc:654] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:24.935818 22179 tablet_bootstrap.cc:492] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782: No bootstrap required, opened a new log
I20240502 02:04:24.936033 22179 ts_tablet_manager.cc:1367] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.000s
I20240502 02:04:24.938380 22179 raft_consensus.cc:348] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } }
I20240502 02:04:24.938535 22179 raft_consensus.cc:374] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:24.938565 22179 raft_consensus.cc:729] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 252575ab96ac4a079bab6ade63792782, State: Initialized, Role: FOLLOWER
I20240502 02:04:24.938920 22183 tablet_bootstrap.cc:492] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0: Bootstrap starting.
I20240502 02:04:24.938980 22179 consensus_queue.cc:260] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } }
I20240502 02:04:24.939806 22179 ts_tablet_manager.cc:1398] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782: Time spent starting tablet: real 0.004s user 0.000s sys 0.004s
I20240502 02:04:24.940095 22179 tablet_bootstrap.cc:492] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782: Bootstrap starting.
I20240502 02:04:24.940982 22183 tablet_bootstrap.cc:654] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:24.941042 22179 tablet_bootstrap.cc:654] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:24.941795 22183 log.cc:824] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:24.942095 22179 tablet_bootstrap.cc:492] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782: No bootstrap required, opened a new log
I20240502 02:04:24.942193 22179 ts_tablet_manager.cc:1367] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 02:04:24.942782 22179 raft_consensus.cc:348] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:24.942867 22178 tablet_bootstrap.cc:492] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0: No bootstrap required, opened a new log
I20240502 02:04:24.942910 22179 raft_consensus.cc:374] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:24.942941 22178 ts_tablet_manager.cc:1367] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.000s
I20240502 02:04:24.942940 22179 raft_consensus.cc:729] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 252575ab96ac4a079bab6ade63792782, State: Initialized, Role: FOLLOWER
I20240502 02:04:24.943079 22179 consensus_queue.cc:260] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:24.943323 22179 ts_tablet_manager.cc:1398] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:24.943475 22179 tablet_bootstrap.cc:492] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782: Bootstrap starting.
I20240502 02:04:24.943436 22178 raft_consensus.cc:348] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:24.943534 22178 raft_consensus.cc:374] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:24.943567 22178 raft_consensus.cc:729] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0983e154926c44ceb508483878dba5a0, State: Initialized, Role: FOLLOWER
I20240502 02:04:24.943711 22178 consensus_queue.cc:260] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:24.944010 22183 tablet_bootstrap.cc:492] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0: No bootstrap required, opened a new log
I20240502 02:04:24.944020 22178 ts_tablet_manager.cc:1398] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:24.944185 22183 ts_tablet_manager.cc:1367] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240502 02:04:24.944227 22178 tablet_bootstrap.cc:492] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: Bootstrap starting.
I20240502 02:04:24.944406 22179 tablet_bootstrap.cc:654] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:24.945108 22178 tablet_bootstrap.cc:654] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:24.946542 22178 tablet_bootstrap.cc:492] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: No bootstrap required, opened a new log
I20240502 02:04:24.946655 22178 ts_tablet_manager.cc:1367] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 02:04:24.947108 22178 raft_consensus.cc:348] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:24.947190 22178 raft_consensus.cc:374] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:24.947227 22178 raft_consensus.cc:729] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0983e154926c44ceb508483878dba5a0, State: Initialized, Role: FOLLOWER
I20240502 02:04:24.947300 22179 tablet_bootstrap.cc:492] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782: No bootstrap required, opened a new log
I20240502 02:04:24.947381 22179 ts_tablet_manager.cc:1367] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 02:04:24.947391 22178 consensus_queue.cc:260] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:24.947628 22178 ts_tablet_manager.cc:1398] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:24.947765 22179 raft_consensus.cc:348] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:24.947883 22179 raft_consensus.cc:374] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:24.947916 22179 raft_consensus.cc:729] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 252575ab96ac4a079bab6ade63792782, State: Initialized, Role: FOLLOWER
I20240502 02:04:24.948030 22179 consensus_queue.cc:260] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:24.948308 22179 ts_tablet_manager.cc:1398] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:24.948961 22183 raft_consensus.cc:348] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:24.949172 22183 raft_consensus.cc:374] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:24.949285 22183 raft_consensus.cc:729] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Initialized, Role: FOLLOWER
I20240502 02:04:24.949748 22183 consensus_queue.cc:260] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:24.950942 22183 ts_tablet_manager.cc:1398] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0: Time spent starting tablet: real 0.007s user 0.006s sys 0.000s
I20240502 02:04:24.951179 22183 tablet_bootstrap.cc:492] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0: Bootstrap starting.
I20240502 02:04:24.952107 22183 tablet_bootstrap.cc:654] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:24.953114 22183 tablet_bootstrap.cc:492] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0: No bootstrap required, opened a new log
I20240502 02:04:24.953217 22183 ts_tablet_manager.cc:1367] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:24.953763 22183 raft_consensus.cc:348] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:24.953864 22183 raft_consensus.cc:374] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:24.953893 22183 raft_consensus.cc:729] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Initialized, Role: FOLLOWER
I20240502 02:04:24.954031 22183 consensus_queue.cc:260] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:24.954267 22183 ts_tablet_manager.cc:1398] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:24.954409 22183 tablet_bootstrap.cc:492] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0: Bootstrap starting.
I20240502 02:04:24.955247 22183 tablet_bootstrap.cc:654] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:24.956354 22183 tablet_bootstrap.cc:492] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0: No bootstrap required, opened a new log
I20240502 02:04:24.956480 22183 ts_tablet_manager.cc:1367] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:24.956859 22183 raft_consensus.cc:348] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } }
I20240502 02:04:24.956941 22183 raft_consensus.cc:374] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:24.956976 22183 raft_consensus.cc:729] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Initialized, Role: FOLLOWER
I20240502 02:04:24.957129 22183 consensus_queue.cc:260] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } }
I20240502 02:04:24.957314 22183 ts_tablet_manager.cc:1398] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
W20240502 02:04:25.031661 21756 tablet.cc:2369] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 02:04:25.062719 22182 raft_consensus.cc:482] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:25.062918 22182 raft_consensus.cc:504] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.064013 22182 leader_election.cc:288] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519), 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219)
I20240502 02:04:25.068265 21440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "15a6b979fd044fe9b58f86ffe137b604" candidate_uuid: "0983e154926c44ceb508483878dba5a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e72358c248634f30be5f9becf0ec0fa0" is_pre_election: true
I20240502 02:04:25.068501 21573 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "15a6b979fd044fe9b58f86ffe137b604" candidate_uuid: "0983e154926c44ceb508483878dba5a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782" is_pre_election: true
I20240502 02:04:25.068614 21440 raft_consensus.cc:2454] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0983e154926c44ceb508483878dba5a0 in term 0.
I20240502 02:04:25.068749 21573 raft_consensus.cc:2454] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0983e154926c44ceb508483878dba5a0 in term 0.
I20240502 02:04:25.069072 21642 leader_election.cc:302] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0983e154926c44ceb508483878dba5a0, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:25.069330 22182 raft_consensus.cc:2781] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:25.069404 22182 raft_consensus.cc:482] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:25.069442 22182 raft_consensus.cc:3037] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.070292 22182 raft_consensus.cc:504] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.070638 22182 leader_election.cc:288] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [CANDIDATE]: Term 1 election: Requested vote from peers e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519), 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219)
I20240502 02:04:25.070994 21573 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "15a6b979fd044fe9b58f86ffe137b604" candidate_uuid: "0983e154926c44ceb508483878dba5a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782"
I20240502 02:04:25.070994 21440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "15a6b979fd044fe9b58f86ffe137b604" candidate_uuid: "0983e154926c44ceb508483878dba5a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e72358c248634f30be5f9becf0ec0fa0"
I20240502 02:04:25.071122 21440 raft_consensus.cc:3037] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.071122 21573 raft_consensus.cc:3037] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.072327 21573 raft_consensus.cc:2454] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0983e154926c44ceb508483878dba5a0 in term 1.
I20240502 02:04:25.072628 21440 raft_consensus.cc:2454] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0983e154926c44ceb508483878dba5a0 in term 1.
I20240502 02:04:25.072634 21640 leader_election.cc:302] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0983e154926c44ceb508483878dba5a0, 252575ab96ac4a079bab6ade63792782; no voters:
I20240502 02:04:25.072796 22182 raft_consensus.cc:2781] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:25.073029 22182 raft_consensus.cc:686] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Becoming Leader. State: Replica: 0983e154926c44ceb508483878dba5a0, State: Running, Role: LEADER
I20240502 02:04:25.073360 22182 consensus_queue.cc:237] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.076409 21296 catalog_manager.cc:5484] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 reported cstate change: term changed from 0 to 1, leader changed from <none> to 0983e154926c44ceb508483878dba5a0 (127.7.221.67). New cstate: current_term: 1 leader_uuid: "0983e154926c44ceb508483878dba5a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:25.079923 22190 raft_consensus.cc:482] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:25.080025 22190 raft_consensus.cc:504] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.080199 22182 raft_consensus.cc:482] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:25.080277 22182 raft_consensus.cc:504] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.080608 22182 leader_election.cc:288] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219), e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519)
I20240502 02:04:25.080938 21440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2fd616a594f245a3b13353f1c9cb9064" candidate_uuid: "0983e154926c44ceb508483878dba5a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e72358c248634f30be5f9becf0ec0fa0" is_pre_election: true
I20240502 02:04:25.081040 21440 raft_consensus.cc:2454] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0983e154926c44ceb508483878dba5a0 in term 0.
I20240502 02:04:25.081179 22190 leader_election.cc:288] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219), 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:25.081284 21573 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2fd616a594f245a3b13353f1c9cb9064" candidate_uuid: "0983e154926c44ceb508483878dba5a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782" is_pre_election: true
I20240502 02:04:25.081413 21573 raft_consensus.cc:2454] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0983e154926c44ceb508483878dba5a0 in term 0.
I20240502 02:04:25.081665 21642 leader_election.cc:302] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0983e154926c44ceb508483878dba5a0, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:25.081838 22182 raft_consensus.cc:2781] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:25.081897 22182 raft_consensus.cc:482] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:25.081949 22182 raft_consensus.cc:3037] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.082957 22182 raft_consensus.cc:504] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.083467 22182 leader_election.cc:288] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [CANDIDATE]: Term 1 election: Requested vote from peers 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219), e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519)
I20240502 02:04:25.083534 21440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2fd616a594f245a3b13353f1c9cb9064" candidate_uuid: "0983e154926c44ceb508483878dba5a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e72358c248634f30be5f9becf0ec0fa0"
I20240502 02:04:25.083638 21440 raft_consensus.cc:3037] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.084388 21440 raft_consensus.cc:2454] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0983e154926c44ceb508483878dba5a0 in term 1.
I20240502 02:04:25.084645 21642 leader_election.cc:302] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0983e154926c44ceb508483878dba5a0, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:25.084811 22182 raft_consensus.cc:2781] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:25.084975 22182 raft_consensus.cc:686] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Becoming Leader. State: Replica: 0983e154926c44ceb508483878dba5a0, State: Running, Role: LEADER
I20240502 02:04:25.085145 22182 consensus_queue.cc:237] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.085505 21573 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2fd616a594f245a3b13353f1c9cb9064" candidate_uuid: "0983e154926c44ceb508483878dba5a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782"
I20240502 02:04:25.085644 21573 raft_consensus.cc:3037] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.085669 21572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2fd616a594f245a3b13353f1c9cb9064" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782" is_pre_election: true
I20240502 02:04:25.086578 21573 raft_consensus.cc:2454] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0983e154926c44ceb508483878dba5a0 in term 1.
I20240502 02:04:25.086912 21708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2fd616a594f245a3b13353f1c9cb9064" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0983e154926c44ceb508483878dba5a0" is_pre_election: true
I20240502 02:04:25.087173 21296 catalog_manager.cc:5484] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 reported cstate change: term changed from 0 to 1, leader changed from <none> to 0983e154926c44ceb508483878dba5a0 (127.7.221.67). New cstate: current_term: 1 leader_uuid: "0983e154926c44ceb508483878dba5a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:25.087455 21375 leader_election.cc:302] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e72358c248634f30be5f9becf0ec0fa0; no voters: 0983e154926c44ceb508483878dba5a0, 252575ab96ac4a079bab6ade63792782
I20240502 02:04:25.087710 22190 raft_consensus.cc:2726] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20240502 02:04:25.114105 21622 tablet.cc:2369] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 02:04:25.156549 22185 raft_consensus.cc:482] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:25.156705 22185 raft_consensus.cc:504] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } }
I20240502 02:04:25.158047 22185 leader_election.cc:288] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557), e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519)
I20240502 02:04:25.161644 21708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1683e87385ab4050824a71508d3c3f5b" candidate_uuid: "252575ab96ac4a079bab6ade63792782" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0983e154926c44ceb508483878dba5a0" is_pre_election: true
I20240502 02:04:25.161648 21440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1683e87385ab4050824a71508d3c3f5b" candidate_uuid: "252575ab96ac4a079bab6ade63792782" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e72358c248634f30be5f9becf0ec0fa0" is_pre_election: true
I20240502 02:04:25.161779 21440 raft_consensus.cc:2454] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 252575ab96ac4a079bab6ade63792782 in term 0.
I20240502 02:04:25.161826 21708 raft_consensus.cc:2454] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 252575ab96ac4a079bab6ade63792782 in term 0.
I20240502 02:04:25.162129 21507 leader_election.cc:302] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 252575ab96ac4a079bab6ade63792782, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:25.162379 22185 raft_consensus.cc:2781] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:25.162442 22185 raft_consensus.cc:482] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:25.162469 22185 raft_consensus.cc:3037] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.163309 22185 raft_consensus.cc:504] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } }
I20240502 02:04:25.163635 22185 leader_election.cc:288] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [CANDIDATE]: Term 1 election: Requested vote from peers 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557), e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519)
I20240502 02:04:25.163949 21440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1683e87385ab4050824a71508d3c3f5b" candidate_uuid: "252575ab96ac4a079bab6ade63792782" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e72358c248634f30be5f9becf0ec0fa0"
I20240502 02:04:25.163954 21708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1683e87385ab4050824a71508d3c3f5b" candidate_uuid: "252575ab96ac4a079bab6ade63792782" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0983e154926c44ceb508483878dba5a0"
I20240502 02:04:25.164047 21440 raft_consensus.cc:3037] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.164053 21708 raft_consensus.cc:3037] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.164759 21440 raft_consensus.cc:2454] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 252575ab96ac4a079bab6ade63792782 in term 1.
I20240502 02:04:25.164911 21708 raft_consensus.cc:2454] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 252575ab96ac4a079bab6ade63792782 in term 1.
I20240502 02:04:25.165047 21507 leader_election.cc:302] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 252575ab96ac4a079bab6ade63792782, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:25.165207 22185 raft_consensus.cc:2781] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:25.165416 22185 raft_consensus.cc:686] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 1 LEADER]: Becoming Leader. State: Replica: 252575ab96ac4a079bab6ade63792782, State: Running, Role: LEADER
I20240502 02:04:25.165702 22185 consensus_queue.cc:237] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } }
I20240502 02:04:25.168527 21296 catalog_manager.cc:5484] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 reported cstate change: term changed from 0 to 1, leader changed from <none> to 252575ab96ac4a079bab6ade63792782 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "252575ab96ac4a079bab6ade63792782" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: UNKNOWN } } }
W20240502 02:04:25.196084 21487 tablet.cc:2369] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 02:04:25.302115 22204 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:25.302356 22204 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:25.305210 22204 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_0'
I20240502 02:04:25.383366 21573 raft_consensus.cc:1261] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:25.383533 21440 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:25.383536 21437 raft_consensus.cc:1261] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:25.383801 21439 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:25.384143 22195 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:25.384294 21573 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:25.384568 22182 consensus_queue.cc:1035] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:25.384779 22185 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:25.384802 22195 consensus_queue.cc:1035] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:25.384872 22182 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:25.389382 21708 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:25.390985 22202 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:25.394656 22234 mvcc.cc:205] Tried to move back new op lower bound from 7023064946204094464 to 7023064944990359552. Current Snapshot: MvccSnapshot[applied={T|T < 7023064946204094464}]
I20240502 02:04:25.398176 22238 mvcc.cc:205] Tried to move back new op lower bound from 7023064946204094464 to 7023064944990359552. Current Snapshot: MvccSnapshot[applied={T|T < 7023064946204094464}]
I20240502 02:04:25.398391 22241 mvcc.cc:205] Tried to move back new op lower bound from 7023064946204094464 to 7023064944990359552. Current Snapshot: MvccSnapshot[applied={T|T < 7023064946204094464}]
INSERT report
rows total: 2000
time total: 101.655 ms
time per row: 0.0508276 ms
I20240502 02:04:25.450996 21295 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51734:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 02:04:25.451469 21295 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_1 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20240502 02:04:25.456100 21419 tablet_service.cc:1467] Processing CreateTablet for tablet 6cd9e4d44fdb40448877d5560a58c9f4 (DEFAULT_TABLE table=rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.456413 21419 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6cd9e4d44fdb40448877d5560a58c9f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.456362 21550 tablet_service.cc:1467] Processing CreateTablet for tablet 6cd9e4d44fdb40448877d5560a58c9f4 (DEFAULT_TABLE table=rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.456634 21550 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6cd9e4d44fdb40448877d5560a58c9f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.458201 21417 tablet_service.cc:1467] Processing CreateTablet for tablet c0db5fac24d54971baf964b55e62abe4 (DEFAULT_TABLE table=rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.458233 21688 tablet_service.cc:1467] Processing CreateTablet for tablet 6cd9e4d44fdb40448877d5560a58c9f4 (DEFAULT_TABLE table=rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.458428 21417 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c0db5fac24d54971baf964b55e62abe4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.458461 21688 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6cd9e4d44fdb40448877d5560a58c9f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.458742 21551 tablet_service.cc:1467] Processing CreateTablet for tablet c0db5fac24d54971baf964b55e62abe4 (DEFAULT_TABLE table=rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.458920 21551 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c0db5fac24d54971baf964b55e62abe4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.460448 22183 tablet_bootstrap.cc:492] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0: Bootstrap starting.
I20240502 02:04:25.461006 22262 tablet_bootstrap.cc:492] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0: Bootstrap starting.
I20240502 02:04:25.460985 21686 tablet_service.cc:1467] Processing CreateTablet for tablet c0db5fac24d54971baf964b55e62abe4 (DEFAULT_TABLE table=rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.460985 21687 tablet_service.cc:1467] Processing CreateTablet for tablet a22071eab1054f0ebc97dfd4e0515148 (DEFAULT_TABLE table=rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.458209 21418 tablet_service.cc:1467] Processing CreateTablet for tablet a22071eab1054f0ebc97dfd4e0515148 (DEFAULT_TABLE table=rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.461216 22183 tablet_bootstrap.cc:654] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.461243 21686 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c0db5fac24d54971baf964b55e62abe4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.461292 21418 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a22071eab1054f0ebc97dfd4e0515148. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.461560 21687 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a22071eab1054f0ebc97dfd4e0515148. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.463142 22183 tablet_bootstrap.cc:492] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0: No bootstrap required, opened a new log
I20240502 02:04:25.463222 22183 ts_tablet_manager.cc:1367] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 02:04:25.463557 22261 tablet_bootstrap.cc:492] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782: Bootstrap starting.
I20240502 02:04:25.463630 22183 raft_consensus.cc:348] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.463727 22183 raft_consensus.cc:374] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.463757 22183 raft_consensus.cc:729] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.463881 22183 consensus_queue.cc:260] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.464141 22183 ts_tablet_manager.cc:1398] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:25.464359 22183 tablet_bootstrap.cc:492] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0: Bootstrap starting.
I20240502 02:04:25.464421 22262 tablet_bootstrap.cc:654] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.464890 22261 tablet_bootstrap.cc:654] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.465241 22183 tablet_bootstrap.cc:654] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.465514 22262 tablet_bootstrap.cc:492] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0: No bootstrap required, opened a new log
I20240502 02:04:25.465586 22262 ts_tablet_manager.cc:1367] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 02:04:25.465894 22261 tablet_bootstrap.cc:492] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782: No bootstrap required, opened a new log
I20240502 02:04:25.465999 22261 ts_tablet_manager.cc:1367] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:25.466012 21551 tablet_service.cc:1467] Processing CreateTablet for tablet a22071eab1054f0ebc97dfd4e0515148 (DEFAULT_TABLE table=rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.465999 22262 raft_consensus.cc:348] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.466101 22262 raft_consensus.cc:374] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.466130 22262 raft_consensus.cc:729] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0983e154926c44ceb508483878dba5a0, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.466236 21551 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a22071eab1054f0ebc97dfd4e0515148. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.466252 22262 consensus_queue.cc:260] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.466437 22261 raft_consensus.cc:348] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.466521 22262 ts_tablet_manager.cc:1398] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:25.466554 22261 raft_consensus.cc:374] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.466636 22261 raft_consensus.cc:729] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 252575ab96ac4a079bab6ade63792782, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.466722 22262 tablet_bootstrap.cc:492] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0: Bootstrap starting.
I20240502 02:04:25.466759 22261 consensus_queue.cc:260] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.467057 22261 ts_tablet_manager.cc:1398] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:25.467226 22261 tablet_bootstrap.cc:492] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: Bootstrap starting.
I20240502 02:04:25.467348 22183 tablet_bootstrap.cc:492] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0: No bootstrap required, opened a new log
I20240502 02:04:25.467407 22183 ts_tablet_manager.cc:1367] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 02:04:25.467546 22262 tablet_bootstrap.cc:654] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.467844 22183 raft_consensus.cc:348] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.467944 22183 raft_consensus.cc:374] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.467978 22183 raft_consensus.cc:729] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.467998 22261 tablet_bootstrap.cc:654] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.468127 22183 consensus_queue.cc:260] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.468771 22183 ts_tablet_manager.cc:1398] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:25.469586 22183 tablet_bootstrap.cc:492] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0: Bootstrap starting.
I20240502 02:04:25.469714 22261 tablet_bootstrap.cc:492] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: No bootstrap required, opened a new log
I20240502 02:04:25.469786 22261 ts_tablet_manager.cc:1367] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 02:04:25.470197 22261 raft_consensus.cc:348] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.470285 22261 raft_consensus.cc:374] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.470314 22261 raft_consensus.cc:729] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 252575ab96ac4a079bab6ade63792782, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.470413 22183 tablet_bootstrap.cc:654] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.470432 22261 consensus_queue.cc:260] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.470691 22261 ts_tablet_manager.cc:1398] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:25.470842 22261 tablet_bootstrap.cc:492] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: Bootstrap starting.
I20240502 02:04:25.471019 22262 tablet_bootstrap.cc:492] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0: No bootstrap required, opened a new log
I20240502 02:04:25.471101 22262 ts_tablet_manager.cc:1367] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 02:04:25.471390 22183 tablet_bootstrap.cc:492] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0: No bootstrap required, opened a new log
I20240502 02:04:25.471462 22183 ts_tablet_manager.cc:1367] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 02:04:25.471500 22262 raft_consensus.cc:348] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.471621 22262 raft_consensus.cc:374] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.471664 22262 raft_consensus.cc:729] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0983e154926c44ceb508483878dba5a0, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.471695 22261 tablet_bootstrap.cc:654] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.471774 22262 consensus_queue.cc:260] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.471886 22183 raft_consensus.cc:348] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.471995 22183 raft_consensus.cc:374] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.472023 22183 raft_consensus.cc:729] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.472059 22262 ts_tablet_manager.cc:1398] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:25.472129 22183 consensus_queue.cc:260] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.472198 22262 tablet_bootstrap.cc:492] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0: Bootstrap starting.
I20240502 02:04:25.472362 22183 ts_tablet_manager.cc:1398] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 02:04:25.472995 22262 tablet_bootstrap.cc:654] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.473445 22261 tablet_bootstrap.cc:492] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: No bootstrap required, opened a new log
I20240502 02:04:25.473539 22261 ts_tablet_manager.cc:1367] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 02:04:25.473919 22261 raft_consensus.cc:348] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.474047 22261 raft_consensus.cc:374] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.474076 22261 raft_consensus.cc:729] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 252575ab96ac4a079bab6ade63792782, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.474086 22262 tablet_bootstrap.cc:492] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0: No bootstrap required, opened a new log
I20240502 02:04:25.474153 22262 ts_tablet_manager.cc:1367] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:25.474185 22261 consensus_queue.cc:260] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.474458 22261 ts_tablet_manager.cc:1398] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:25.474876 22262 raft_consensus.cc:348] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.474967 22262 raft_consensus.cc:374] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.475003 22262 raft_consensus.cc:729] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0983e154926c44ceb508483878dba5a0, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.475098 22262 consensus_queue.cc:260] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.475363 22262 ts_tablet_manager.cc:1398] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:25.492378 22236 raft_consensus.cc:482] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:25.492496 22236 raft_consensus.cc:504] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.492868 22236 leader_election.cc:288] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219), 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:25.493247 21572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c0db5fac24d54971baf964b55e62abe4" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782" is_pre_election: true
I20240502 02:04:25.493250 21708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c0db5fac24d54971baf964b55e62abe4" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0983e154926c44ceb508483878dba5a0" is_pre_election: true
I20240502 02:04:25.493397 21708 raft_consensus.cc:2454] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 0.
I20240502 02:04:25.493391 21572 raft_consensus.cc:2454] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 0.
I20240502 02:04:25.493696 21375 leader_election.cc:302] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0983e154926c44ceb508483878dba5a0, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:25.493896 22236 raft_consensus.cc:2781] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:25.493948 22236 raft_consensus.cc:482] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:25.493985 22236 raft_consensus.cc:3037] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.494956 22236 raft_consensus.cc:504] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.495220 22236 leader_election.cc:288] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 election: Requested vote from peers 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219), 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:25.495532 21572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c0db5fac24d54971baf964b55e62abe4" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782"
I20240502 02:04:25.495543 21708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c0db5fac24d54971baf964b55e62abe4" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0983e154926c44ceb508483878dba5a0"
I20240502 02:04:25.495642 21572 raft_consensus.cc:3037] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.495669 21708 raft_consensus.cc:3037] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.496415 21572 raft_consensus.cc:2454] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 1.
I20240502 02:04:25.496426 21708 raft_consensus.cc:2454] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 1.
I20240502 02:04:25.496716 21372 leader_election.cc:302] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 252575ab96ac4a079bab6ade63792782, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:25.496901 22236 raft_consensus.cc:2781] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:25.496994 22236 raft_consensus.cc:686] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Becoming Leader. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Running, Role: LEADER
I20240502 02:04:25.497298 22236 consensus_queue.cc:237] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.499648 21293 catalog_manager.cc:5484] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: term changed from 0 to 1, leader changed from <none> to e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:25.540935 22236 raft_consensus.cc:482] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:25.541098 22236 raft_consensus.cc:504] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.541592 22236 leader_election.cc:288] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219), 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:25.541970 21708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a22071eab1054f0ebc97dfd4e0515148" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0983e154926c44ceb508483878dba5a0" is_pre_election: true
I20240502 02:04:25.542042 21572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a22071eab1054f0ebc97dfd4e0515148" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782" is_pre_election: true
I20240502 02:04:25.542125 21708 raft_consensus.cc:2454] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 0.
I20240502 02:04:25.542200 21572 raft_consensus.cc:2454] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 0.
I20240502 02:04:25.542387 21375 leader_election.cc:302] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0983e154926c44ceb508483878dba5a0, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:25.542583 22236 raft_consensus.cc:2781] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:25.542678 22236 raft_consensus.cc:482] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:25.542729 22236 raft_consensus.cc:3037] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.543615 22236 raft_consensus.cc:504] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.543963 22236 leader_election.cc:288] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 election: Requested vote from peers 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219), 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:25.544292 21572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a22071eab1054f0ebc97dfd4e0515148" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782"
I20240502 02:04:25.544412 21572 raft_consensus.cc:3037] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.544417 21708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a22071eab1054f0ebc97dfd4e0515148" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0983e154926c44ceb508483878dba5a0"
I20240502 02:04:25.544517 21708 raft_consensus.cc:3037] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.545158 21572 raft_consensus.cc:2454] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 1.
I20240502 02:04:25.545336 21708 raft_consensus.cc:2454] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 1.
I20240502 02:04:25.545491 21372 leader_election.cc:302] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 252575ab96ac4a079bab6ade63792782, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:25.545650 22236 raft_consensus.cc:2781] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:25.545743 22236 raft_consensus.cc:686] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Becoming Leader. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Running, Role: LEADER
I20240502 02:04:25.545905 22236 consensus_queue.cc:237] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.547544 21293 catalog_manager.cc:5484] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: term changed from 0 to 1, leader changed from <none> to e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:25.558547 22236 raft_consensus.cc:482] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:25.558702 22236 raft_consensus.cc:504] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.559087 22236 leader_election.cc:288] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219), 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:25.559509 21572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6cd9e4d44fdb40448877d5560a58c9f4" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782" is_pre_election: true
I20240502 02:04:25.559509 21708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6cd9e4d44fdb40448877d5560a58c9f4" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0983e154926c44ceb508483878dba5a0" is_pre_election: true
I20240502 02:04:25.559639 21708 raft_consensus.cc:2454] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 0.
I20240502 02:04:25.559655 21572 raft_consensus.cc:2454] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 0.
I20240502 02:04:25.559937 21375 leader_election.cc:302] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0983e154926c44ceb508483878dba5a0, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:25.560120 22236 raft_consensus.cc:2781] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:25.560178 22236 raft_consensus.cc:482] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:25.560214 22236 raft_consensus.cc:3037] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.561058 22236 raft_consensus.cc:504] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.561367 22236 leader_election.cc:288] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 election: Requested vote from peers 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219), 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:25.561661 21572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6cd9e4d44fdb40448877d5560a58c9f4" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782"
I20240502 02:04:25.561699 21708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6cd9e4d44fdb40448877d5560a58c9f4" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0983e154926c44ceb508483878dba5a0"
I20240502 02:04:25.561782 21708 raft_consensus.cc:3037] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.561789 21572 raft_consensus.cc:3037] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.562474 21708 raft_consensus.cc:2454] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 1.
I20240502 02:04:25.562490 21572 raft_consensus.cc:2454] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 1.
I20240502 02:04:25.562732 21375 leader_election.cc:302] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0983e154926c44ceb508483878dba5a0, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:25.562897 22236 raft_consensus.cc:2781] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:25.562984 22236 raft_consensus.cc:686] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Becoming Leader. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Running, Role: LEADER
I20240502 02:04:25.563127 22236 consensus_queue.cc:237] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.564667 21293 catalog_manager.cc:5484] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: term changed from 0 to 1, leader changed from <none> to e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: UNKNOWN } } }
W20240502 02:04:25.685889 22276 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:25.686136 22276 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:25.688899 22276 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_1'
I20240502 02:04:25.762107 21708 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:25.762859 21572 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:25.763312 22236 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:25.763793 21708 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:25.763818 22236 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:25.763904 21707 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:25.764608 21573 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:25.764842 21572 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:25.765014 22236 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:25.765265 22236 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:25.767721 22190 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:25.768005 22190 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 94.9049 ms
time per row: 0.0474525 ms
I20240502 02:04:25.822391 21292 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51734:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 02:04:25.822911 21292 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_2 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20240502 02:04:25.827679 21418 tablet_service.cc:1467] Processing CreateTablet for tablet fd1b849d9ecd4f5e96b678673444f4d6 (DEFAULT_TABLE table=rebalance_test_table_2 [id=c5c742d762c44789b376f2c4c64c13b2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.827857 21687 tablet_service.cc:1467] Processing CreateTablet for tablet fd1b849d9ecd4f5e96b678673444f4d6 (DEFAULT_TABLE table=rebalance_test_table_2 [id=c5c742d762c44789b376f2c4c64c13b2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.827984 21418 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fd1b849d9ecd4f5e96b678673444f4d6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.828125 21687 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fd1b849d9ecd4f5e96b678673444f4d6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.828176 21550 tablet_service.cc:1467] Processing CreateTablet for tablet bb5988c6c4c14f46a5e598cc1a59c99b (DEFAULT_TABLE table=rebalance_test_table_2 [id=c5c742d762c44789b376f2c4c64c13b2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.828296 21686 tablet_service.cc:1467] Processing CreateTablet for tablet bb5988c6c4c14f46a5e598cc1a59c99b (DEFAULT_TABLE table=rebalance_test_table_2 [id=c5c742d762c44789b376f2c4c64c13b2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.828406 21550 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bb5988c6c4c14f46a5e598cc1a59c99b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.828490 21686 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bb5988c6c4c14f46a5e598cc1a59c99b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.830193 21551 tablet_service.cc:1467] Processing CreateTablet for tablet fd1b849d9ecd4f5e96b678673444f4d6 (DEFAULT_TABLE table=rebalance_test_table_2 [id=c5c742d762c44789b376f2c4c64c13b2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.830266 21688 tablet_service.cc:1467] Processing CreateTablet for tablet fe41f0bd0a884eaa9def3489cece439f (DEFAULT_TABLE table=rebalance_test_table_2 [id=c5c742d762c44789b376f2c4c64c13b2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.830407 21551 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fd1b849d9ecd4f5e96b678673444f4d6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.830488 21688 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fe41f0bd0a884eaa9def3489cece439f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.830567 22183 tablet_bootstrap.cc:492] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: Bootstrap starting.
I20240502 02:04:25.831386 22183 tablet_bootstrap.cc:654] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.831952 22262 tablet_bootstrap.cc:492] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0: Bootstrap starting.
I20240502 02:04:25.832266 21417 tablet_service.cc:1467] Processing CreateTablet for tablet fe41f0bd0a884eaa9def3489cece439f (DEFAULT_TABLE table=rebalance_test_table_2 [id=c5c742d762c44789b376f2c4c64c13b2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.832490 21417 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fe41f0bd0a884eaa9def3489cece439f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.832733 22262 tablet_bootstrap.cc:654] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.832688 21418 tablet_service.cc:1467] Processing CreateTablet for tablet bb5988c6c4c14f46a5e598cc1a59c99b (DEFAULT_TABLE table=rebalance_test_table_2 [id=c5c742d762c44789b376f2c4c64c13b2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.832885 21418 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bb5988c6c4c14f46a5e598cc1a59c99b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.833324 22261 tablet_bootstrap.cc:492] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782: Bootstrap starting.
I20240502 02:04:25.834260 22261 tablet_bootstrap.cc:654] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.835201 22183 tablet_bootstrap.cc:492] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: No bootstrap required, opened a new log
I20240502 02:04:25.835301 22183 ts_tablet_manager.cc:1367] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.001s
I20240502 02:04:25.835472 22262 tablet_bootstrap.cc:492] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0: No bootstrap required, opened a new log
I20240502 02:04:25.835474 21551 tablet_service.cc:1467] Processing CreateTablet for tablet fe41f0bd0a884eaa9def3489cece439f (DEFAULT_TABLE table=rebalance_test_table_2 [id=c5c742d762c44789b376f2c4c64c13b2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:25.835556 22262 ts_tablet_manager.cc:1367] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 02:04:25.835666 21551 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fe41f0bd0a884eaa9def3489cece439f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:25.835685 22183 raft_consensus.cc:348] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.835784 22183 raft_consensus.cc:374] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.835820 22183 raft_consensus.cc:729] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.835947 22183 consensus_queue.cc:260] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.835999 22262 raft_consensus.cc:348] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.836107 22262 raft_consensus.cc:374] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.836141 22262 raft_consensus.cc:729] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0983e154926c44ceb508483878dba5a0, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.836230 22183 ts_tablet_manager.cc:1398] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 02:04:25.836270 22262 consensus_queue.cc:260] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.836398 22183 tablet_bootstrap.cc:492] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0: Bootstrap starting.
I20240502 02:04:25.836540 22262 ts_tablet_manager.cc:1398] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:25.836705 22262 tablet_bootstrap.cc:492] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0: Bootstrap starting.
I20240502 02:04:25.837232 22183 tablet_bootstrap.cc:654] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.837551 22262 tablet_bootstrap.cc:654] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.838871 22183 tablet_bootstrap.cc:492] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0: No bootstrap required, opened a new log
I20240502 02:04:25.838891 22261 tablet_bootstrap.cc:492] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782: No bootstrap required, opened a new log
I20240502 02:04:25.838953 22183 ts_tablet_manager.cc:1367] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 02:04:25.838964 22261 ts_tablet_manager.cc:1367] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 02:04:25.839295 22261 raft_consensus.cc:348] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.839401 22261 raft_consensus.cc:374] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.839433 22261 raft_consensus.cc:729] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 252575ab96ac4a079bab6ade63792782, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.839385 22183 raft_consensus.cc:348] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.839468 22183 raft_consensus.cc:374] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.839506 22183 raft_consensus.cc:729] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.839543 22261 consensus_queue.cc:260] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.839607 22183 consensus_queue.cc:260] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.839819 22261 ts_tablet_manager.cc:1398] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:25.839880 22183 ts_tablet_manager.cc:1398] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 02:04:25.839980 22261 tablet_bootstrap.cc:492] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782: Bootstrap starting.
I20240502 02:04:25.840020 22183 tablet_bootstrap.cc:492] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0: Bootstrap starting.
I20240502 02:04:25.840744 22183 tablet_bootstrap.cc:654] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.840766 22261 tablet_bootstrap.cc:654] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.841889 22262 tablet_bootstrap.cc:492] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0: No bootstrap required, opened a new log
I20240502 02:04:25.841970 22262 ts_tablet_manager.cc:1367] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 02:04:25.842233 22183 tablet_bootstrap.cc:492] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0: No bootstrap required, opened a new log
I20240502 02:04:25.842288 22183 ts_tablet_manager.cc:1367] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 02:04:25.842422 22262 raft_consensus.cc:348] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.842514 22262 raft_consensus.cc:374] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.842550 22262 raft_consensus.cc:729] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0983e154926c44ceb508483878dba5a0, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.842547 22183 raft_consensus.cc:348] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.842653 22183 raft_consensus.cc:374] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.842686 22183 raft_consensus.cc:729] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.842681 22262 consensus_queue.cc:260] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.842784 22183 consensus_queue.cc:260] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.842962 22262 ts_tablet_manager.cc:1398] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 02:04:25.842989 22183 ts_tablet_manager.cc:1398] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 02:04:25.843106 22262 tablet_bootstrap.cc:492] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0: Bootstrap starting.
I20240502 02:04:25.843667 22261 tablet_bootstrap.cc:492] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782: No bootstrap required, opened a new log
I20240502 02:04:25.843742 22261 ts_tablet_manager.cc:1367] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 02:04:25.844096 22262 tablet_bootstrap.cc:654] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.845031 22261 raft_consensus.cc:348] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.845152 22261 raft_consensus.cc:374] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.845201 22261 raft_consensus.cc:729] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 252575ab96ac4a079bab6ade63792782, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.845332 22261 consensus_queue.cc:260] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.845531 22262 tablet_bootstrap.cc:492] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0: No bootstrap required, opened a new log
I20240502 02:04:25.845614 22262 ts_tablet_manager.cc:1367] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 02:04:25.845661 22261 ts_tablet_manager.cc:1398] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:25.845813 22261 tablet_bootstrap.cc:492] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782: Bootstrap starting.
I20240502 02:04:25.846014 22262 raft_consensus.cc:348] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.846108 22262 raft_consensus.cc:374] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.846143 22262 raft_consensus.cc:729] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0983e154926c44ceb508483878dba5a0, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.846277 22262 consensus_queue.cc:260] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.846647 22262 ts_tablet_manager.cc:1398] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 02:04:25.846767 22261 tablet_bootstrap.cc:654] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:25.847808 22261 tablet_bootstrap.cc:492] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782: No bootstrap required, opened a new log
I20240502 02:04:25.847908 22261 ts_tablet_manager.cc:1367] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 02:04:25.848287 22261 raft_consensus.cc:348] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.848387 22261 raft_consensus.cc:374] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:25.848430 22261 raft_consensus.cc:729] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 252575ab96ac4a079bab6ade63792782, State: Initialized, Role: FOLLOWER
I20240502 02:04:25.848552 22261 consensus_queue.cc:260] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.848826 22261 ts_tablet_manager.cc:1398] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:25.860464 22236 raft_consensus.cc:482] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:25.860544 22236 raft_consensus.cc:504] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.860889 22236 leader_election.cc:288] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219), 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:25.861186 21573 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782" is_pre_election: true
I20240502 02:04:25.861222 21706 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0983e154926c44ceb508483878dba5a0" is_pre_election: true
I20240502 02:04:25.861295 21573 raft_consensus.cc:2454] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 0.
I20240502 02:04:25.861327 21706 raft_consensus.cc:2454] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 0.
I20240502 02:04:25.861559 21372 leader_election.cc:302] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 252575ab96ac4a079bab6ade63792782, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:25.861716 22236 raft_consensus.cc:2781] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:25.861774 22236 raft_consensus.cc:482] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:25.861821 22236 raft_consensus.cc:3037] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.862684 22236 raft_consensus.cc:504] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.862993 22236 leader_election.cc:288] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 election: Requested vote from peers 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219), 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:25.863277 21573 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782"
I20240502 02:04:25.863289 21706 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0983e154926c44ceb508483878dba5a0"
I20240502 02:04:25.863373 21573 raft_consensus.cc:3037] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.863399 21706 raft_consensus.cc:3037] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:25.864128 21573 raft_consensus.cc:2454] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 1.
I20240502 02:04:25.864189 21706 raft_consensus.cc:2454] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 1.
I20240502 02:04:25.864456 21372 leader_election.cc:302] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 252575ab96ac4a079bab6ade63792782, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:25.864634 22236 raft_consensus.cc:2781] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:25.864718 22236 raft_consensus.cc:686] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Becoming Leader. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Running, Role: LEADER
I20240502 02:04:25.864843 22236 consensus_queue.cc:237] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:25.866251 21292 catalog_manager.cc:5484] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: term changed from 0 to 1, leader changed from <none> to e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:26.002926 22254 raft_consensus.cc:482] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:26.003068 22254 raft_consensus.cc:504] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:26.003512 22254 leader_election.cc:288] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519), 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219)
I20240502 02:04:26.003947 21573 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fd1b849d9ecd4f5e96b678673444f4d6" candidate_uuid: "0983e154926c44ceb508483878dba5a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782" is_pre_election: true
I20240502 02:04:26.003948 21440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fd1b849d9ecd4f5e96b678673444f4d6" candidate_uuid: "0983e154926c44ceb508483878dba5a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e72358c248634f30be5f9becf0ec0fa0" is_pre_election: true
I20240502 02:04:26.004086 21573 raft_consensus.cc:2454] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0983e154926c44ceb508483878dba5a0 in term 0.
I20240502 02:04:26.004106 21440 raft_consensus.cc:2454] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0983e154926c44ceb508483878dba5a0 in term 0.
I20240502 02:04:26.004436 21640 leader_election.cc:302] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0983e154926c44ceb508483878dba5a0, 252575ab96ac4a079bab6ade63792782; no voters:
I20240502 02:04:26.004609 22254 raft_consensus.cc:2781] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:26.004660 22254 raft_consensus.cc:482] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:26.004698 22254 raft_consensus.cc:3037] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:26.005532 22254 raft_consensus.cc:504] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:26.005841 22254 leader_election.cc:288] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [CANDIDATE]: Term 1 election: Requested vote from peers e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519), 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219)
I20240502 02:04:26.006160 21440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fd1b849d9ecd4f5e96b678673444f4d6" candidate_uuid: "0983e154926c44ceb508483878dba5a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e72358c248634f30be5f9becf0ec0fa0"
I20240502 02:04:26.006215 21573 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fd1b849d9ecd4f5e96b678673444f4d6" candidate_uuid: "0983e154926c44ceb508483878dba5a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782"
I20240502 02:04:26.006273 21440 raft_consensus.cc:3037] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:26.006310 21573 raft_consensus.cc:3037] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:26.007072 21573 raft_consensus.cc:2454] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0983e154926c44ceb508483878dba5a0 in term 1.
I20240502 02:04:26.007099 21440 raft_consensus.cc:2454] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0983e154926c44ceb508483878dba5a0 in term 1.
I20240502 02:04:26.007360 21640 leader_election.cc:302] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0983e154926c44ceb508483878dba5a0, 252575ab96ac4a079bab6ade63792782; no voters:
I20240502 02:04:26.007520 22254 raft_consensus.cc:2781] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:26.007611 22254 raft_consensus.cc:686] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Becoming Leader. State: Replica: 0983e154926c44ceb508483878dba5a0, State: Running, Role: LEADER
I20240502 02:04:26.007753 22254 consensus_queue.cc:237] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:26.009344 21292 catalog_manager.cc:5484] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 reported cstate change: term changed from 0 to 1, leader changed from <none> to 0983e154926c44ceb508483878dba5a0 (127.7.221.67). New cstate: current_term: 1 leader_uuid: "0983e154926c44ceb508483878dba5a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:26.025200 22236 raft_consensus.cc:482] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:26.025317 22236 raft_consensus.cc:504] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:26.025734 22236 leader_election.cc:288] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219), 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:26.026189 21573 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fe41f0bd0a884eaa9def3489cece439f" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782" is_pre_election: true
I20240502 02:04:26.026198 21706 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fe41f0bd0a884eaa9def3489cece439f" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0983e154926c44ceb508483878dba5a0" is_pre_election: true
I20240502 02:04:26.026335 21573 raft_consensus.cc:2454] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 0.
I20240502 02:04:26.026341 21706 raft_consensus.cc:2454] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 0.
I20240502 02:04:26.026675 21375 leader_election.cc:302] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0983e154926c44ceb508483878dba5a0, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:26.026865 22236 raft_consensus.cc:2781] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:26.026934 22236 raft_consensus.cc:482] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:26.026986 22236 raft_consensus.cc:3037] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:26.027729 22236 raft_consensus.cc:504] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:26.028056 22236 leader_election.cc:288] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 election: Requested vote from peers 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219), 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:26.028350 21706 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fe41f0bd0a884eaa9def3489cece439f" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0983e154926c44ceb508483878dba5a0"
I20240502 02:04:26.028407 21573 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fe41f0bd0a884eaa9def3489cece439f" candidate_uuid: "e72358c248634f30be5f9becf0ec0fa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "252575ab96ac4a079bab6ade63792782"
I20240502 02:04:26.028493 21706 raft_consensus.cc:3037] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:26.028501 21573 raft_consensus.cc:3037] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:26.029212 21573 raft_consensus.cc:2454] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 1.
I20240502 02:04:26.029253 21706 raft_consensus.cc:2454] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e72358c248634f30be5f9becf0ec0fa0 in term 1.
I20240502 02:04:26.029500 21372 leader_election.cc:302] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 252575ab96ac4a079bab6ade63792782, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:26.029675 22236 raft_consensus.cc:2781] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:26.029764 22236 raft_consensus.cc:686] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Becoming Leader. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Running, Role: LEADER
I20240502 02:04:26.029924 22236 consensus_queue.cc:237] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } }
I20240502 02:04:26.031388 21292 catalog_manager.cc:5484] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: term changed from 0 to 1, leader changed from <none> to e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65). New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: UNKNOWN } } }
W20240502 02:04:26.175097 22324 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:26.175324 22324 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:26.178229 22324 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_2'
I20240502 02:04:26.272980 21573 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:26.273077 21571 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:26.273195 21440 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:26.273348 21708 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:26.273475 22254 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:26.273614 22235 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:26.275724 22304 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:26.275910 22190 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:26.276110 21571 raft_consensus.cc:1261] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:26.282667 21708 raft_consensus.cc:1261] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:26.284113 22304 consensus_queue.cc:1035] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:26.300359 22356 consensus_queue.cc:1035] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 114.678 ms
time per row: 0.0573388 ms
I20240502 02:04:26.339712 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.69:45619
--local_ip_for_outbound_sockets=127.7.221.69
--tserver_master_addrs=127.7.221.126:38635
--webserver_port=44923
--webserver_interface=127.7.221.69
--builtin_ntp_servers=127.7.221.84:44609
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 02:04:26.451866 22364 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:26.452049 22364 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:26.452111 22364 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:26.454807 22364 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:26.454882 22364 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:26.454954 22364 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.69
I20240502 02:04:26.458214 22364 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:44609
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.69:45619
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.7.221.69
--webserver_port=44923
--tserver_master_addrs=127.7.221.126:38635
--never_fsync=true
--heap_profile_path=/tmp/kudu.22364
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.69
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:26.458992 22364 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:26.459968 22364 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:26.467878 22371 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:26.468030 22370 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:26.468734 22364 server_base.cc:1034] running on GCE node
W20240502 02:04:26.469233 22373 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:26.469574 22364 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:26.470079 22364 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:26.471215 22364 hybrid_clock.cc:648] HybridClock initialized: now 1714615466471191 us; error 35 us; skew 500 ppm
I20240502 02:04:26.472795 22364 webserver.cc:460] Webserver started at http://127.7.221.69:44923/ using document root <none> and password file <none>
I20240502 02:04:26.473268 22364 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:26.473335 22364 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:26.476317 22364 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.003s sys 0.000s
I20240502 02:04:26.478075 22379 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:26.478948 22364 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:26.479038 22364 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "5e9df19de7264d1a9ee930ee770a8701"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:26.479401 22364 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:26.506700 22364 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:26.507256 22364 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:26.507396 22364 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:26.507874 22364 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:26.508637 22364 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:26.508680 22364 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:26.508733 22364 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:26.508756 22364 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:26.516476 22364 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.69:45619
I20240502 02:04:26.516534 22494 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.69:45619 every 8 connection(s)
I20240502 02:04:26.517444 22364 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 02:04:26.525117 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 22364
I20240502 02:04:26.525339 22495 heartbeater.cc:347] Connected to a master server at 127.7.221.126:38635
I20240502 02:04:26.525553 22495 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:26.525552 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.70:39691
--local_ip_for_outbound_sockets=127.7.221.70
--tserver_master_addrs=127.7.221.126:38635
--webserver_port=37405
--webserver_interface=127.7.221.70
--builtin_ntp_servers=127.7.221.84:44609
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 02:04:26.526108 22495 heartbeater.cc:511] Master 127.7.221.126:38635 requested a full tablet report, sending...
I20240502 02:04:26.527339 21292 ts_manager.cc:194] Re-registered known tserver with Master: 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69:45619)
I20240502 02:04:26.528092 21292 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.69:54595
W20240502 02:04:26.635901 22498 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:26.636124 22498 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:26.636168 22498 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:26.638875 22498 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:26.638926 22498 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:26.638999 22498 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.70
I20240502 02:04:26.642185 22498 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:44609
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.70:39691
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.7.221.70
--webserver_port=37405
--tserver_master_addrs=127.7.221.126:38635
--never_fsync=true
--heap_profile_path=/tmp/kudu.22498
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.70
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:26.643102 22498 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:26.644119 22498 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:26.651680 22504 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:26.652206 22506 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:26.652885 22508 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:26.653543 22498 server_base.cc:1034] running on GCE node
I20240502 02:04:26.653904 22498 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:26.654487 22498 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:26.655638 22498 hybrid_clock.cc:648] HybridClock initialized: now 1714615466655630 us; error 25 us; skew 500 ppm
I20240502 02:04:26.657442 22498 webserver.cc:460] Webserver started at http://127.7.221.70:37405/ using document root <none> and password file <none>
I20240502 02:04:26.657966 22498 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:26.658038 22498 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:26.661176 22498 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:26.662931 22514 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:26.663820 22498 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:26.663918 22498 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "2590825fe67b48c9adb5894c322237dc"
format_stamp: "Formatted at 2024-05-02 02:04:21 on dist-test-slave-dmxt"
I20240502 02:04:26.664350 22498 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:26.679389 22498 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:26.679965 22498 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:26.680115 22498 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:26.680627 22498 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:26.681478 22498 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:26.681524 22498 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:26.681566 22498 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:26.681598 22498 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:26.689988 22498 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.70:39691
I20240502 02:04:26.690022 22629 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.70:39691 every 8 connection(s)
I20240502 02:04:26.691074 22498 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 02:04:26.696507 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 22498
I20240502 02:04:26.696979 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.68:46455
--local_ip_for_outbound_sockets=127.7.221.68
--tserver_master_addrs=127.7.221.126:38635
--webserver_port=42765
--webserver_interface=127.7.221.68
--builtin_ntp_servers=127.7.221.84:44609
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 02:04:26.699884 22630 heartbeater.cc:347] Connected to a master server at 127.7.221.126:38635
I20240502 02:04:26.700085 22630 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:26.700547 22630 heartbeater.cc:511] Master 127.7.221.126:38635 requested a full tablet report, sending...
I20240502 02:04:26.701599 21292 ts_manager.cc:194] Re-registered known tserver with Master: 2590825fe67b48c9adb5894c322237dc (127.7.221.70:39691)
I20240502 02:04:26.702358 21292 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.70:58765
W20240502 02:04:26.800498 22633 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:26.800688 22633 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:26.800743 22633 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:26.803346 22633 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:26.803401 22633 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:26.803459 22633 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.68
I20240502 02:04:26.806938 22633 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:44609
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.68:46455
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.7.221.68
--webserver_port=42765
--tserver_master_addrs=127.7.221.126:38635
--never_fsync=true
--heap_profile_path=/tmp/kudu.22633
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.68
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:26.807817 22633 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:26.809010 22633 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:26.816498 22638 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:26.817147 22639 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:26.817902 22643 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:26.818483 22633 server_base.cc:1034] running on GCE node
I20240502 02:04:26.818878 22633 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:26.819474 22633 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:26.820636 22633 hybrid_clock.cc:648] HybridClock initialized: now 1714615466820613 us; error 41 us; skew 500 ppm
I20240502 02:04:26.822325 22633 webserver.cc:460] Webserver started at http://127.7.221.68:42765/ using document root <none> and password file <none>
I20240502 02:04:26.822924 22633 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:26.823009 22633 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:26.826030 22633 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.003s sys 0.000s
I20240502 02:04:26.827970 22649 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:26.828855 22633 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:26.828941 22633 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "d590702730234f799e2a26de695bd7a2"
format_stamp: "Formatted at 2024-05-02 02:04:20 on dist-test-slave-dmxt"
I20240502 02:04:26.829311 22633 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:26.862340 22633 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:26.863003 22633 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:26.863168 22633 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:26.863790 22633 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:26.864702 22633 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:26.864765 22633 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:26.864811 22633 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:26.864832 22633 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:26.873603 22633 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.68:46455
I20240502 02:04:26.873636 22763 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.68:46455 every 8 connection(s)
I20240502 02:04:26.874580 22633 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 02:04:26.882475 22764 heartbeater.cc:347] Connected to a master server at 127.7.221.126:38635
I20240502 02:04:26.882725 22764 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:26.883221 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 22633
I20240502 02:04:26.883294 22764 heartbeater.cc:511] Master 127.7.221.126:38635 requested a full tablet report, sending...
I20240502 02:04:26.885749 21292 ts_manager.cc:194] Re-registered known tserver with Master: d590702730234f799e2a26de695bd7a2 (127.7.221.68:46455)
I20240502 02:04:26.886370 21292 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.68:42583
I20240502 02:04:27.063741 22562 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:27.064011 21687 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:27.064278 21551 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:27.067306 22428 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:27.077113 21417 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 02:04:27.085624 22697 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:27.164487 22562 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:27.167085 21551 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:27.167270 21687 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:27.174006 22697 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:27.184084 22428 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:27.192047 21417 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 02:04:27.282855 22428 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:27.284786 22562 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:27.286924 22697 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:27.289407 21417 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 02:04:27.291409 21687 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:27.300946 21551 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:27.376792 22562 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:27.378108 21687 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:27.381268 22697 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:27.382150 21417 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 02:04:27.382570 22428 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:27.389964 21551 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:27.428437 21571 consensus_queue.cc:237] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } }
I20240502 02:04:27.429968 21440 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:27.430022 21708 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:27.430378 22185 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:27.430583 22925 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:27.432862 22185 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } } }
W20240502 02:04:27.434314 21506 consensus_peers.cc:485] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 -> Peer 2590825fe67b48c9adb5894c322237dc (127.7.221.70:39691): Couldn't send request to peer 2590825fe67b48c9adb5894c322237dc. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1683e87385ab4050824a71508d3c3f5b. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:27.434970 21440 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } } }
I20240502 02:04:27.435129 21706 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } } }
I20240502 02:04:27.435834 21292 catalog_manager.cc:5484] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index -1 to 4, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "252575ab96ac4a079bab6ade63792782" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:27.443681 21440 consensus_queue.cc:237] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } }
I20240502 02:04:27.445050 21708 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:27.445302 21571 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:27.445463 22940 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:27.445700 22190 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:27.447474 22939 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:27.448205 21708 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
W20240502 02:04:27.449378 21372 consensus_peers.cc:485] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 -> Peer 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69:45619): Couldn't send request to peer 5e9df19de7264d1a9ee930ee770a8701. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a22071eab1054f0ebc97dfd4e0515148. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:27.450176 21571 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:27.450692 21295 catalog_manager.cc:5484] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 reported cstate change: config changed from index -1 to 4, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:27.458348 21708 consensus_queue.cc:237] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } }
I20240502 02:04:27.459780 21571 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:27.459942 21440 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:27.460187 22254 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:27.460371 22951 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:27.461902 22254 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } } }
I20240502 02:04:27.462846 21440 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } } }
W20240502 02:04:27.463749 21642 consensus_peers.cc:485] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 -> Peer d590702730234f799e2a26de695bd7a2 (127.7.221.68:46455): Couldn't send request to peer d590702730234f799e2a26de695bd7a2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fd1b849d9ecd4f5e96b678673444f4d6. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:27.464057 21571 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } } }
I20240502 02:04:27.464185 21292 catalog_manager.cc:5484] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 reported cstate change: config changed from index -1 to 4, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "0983e154926c44ceb508483878dba5a0" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:27.471961 21440 consensus_queue.cc:237] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } }
I20240502 02:04:27.473222 21571 raft_consensus.cc:1261] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:27.473345 21708 raft_consensus.cc:1261] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:27.473564 22939 consensus_queue.cc:1035] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:27.474120 22939 consensus_queue.cc:1035] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:27.475265 22940 raft_consensus.cc:2932] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } } }
I20240502 02:04:27.475704 21571 raft_consensus.cc:2932] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } } }
I20240502 02:04:27.476099 21708 raft_consensus.cc:2932] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } } }
W20240502 02:04:27.477288 21373 consensus_peers.cc:485] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 -> Peer 2590825fe67b48c9adb5894c322237dc (127.7.221.70:39691): Couldn't send request to peer 2590825fe67b48c9adb5894c322237dc. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fe41f0bd0a884eaa9def3489cece439f. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:27.478042 21292 catalog_manager.cc:5484] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index -1 to 4, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } } }
I20240502 02:04:27.484998 21440 consensus_queue.cc:237] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } }
I20240502 02:04:27.486269 21571 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 02:04:27.486496 21372 consensus_peers.cc:485] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 -> Peer 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69:45619): Couldn't send request to peer 5e9df19de7264d1a9ee930ee770a8701. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c0db5fac24d54971baf964b55e62abe4. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:27.486647 21708 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:27.486759 22939 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:27.486964 22940 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:27.488416 22939 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:27.488972 21571 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:27.489089 21708 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:27.491235 21294 catalog_manager.cc:5484] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index -1 to 4, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:27.497831 21708 consensus_queue.cc:237] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } }
I20240502 02:04:27.499030 21440 raft_consensus.cc:1261] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:27.499076 21571 raft_consensus.cc:1261] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 02:04:27.499256 21642 consensus_peers.cc:485] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 -> Peer d590702730234f799e2a26de695bd7a2 (127.7.221.68:46455): Couldn't send request to peer d590702730234f799e2a26de695bd7a2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 15a6b979fd044fe9b58f86ffe137b604. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:27.499403 22951 consensus_queue.cc:1035] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:27.499569 22950 consensus_queue.cc:1035] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:27.501065 22956 raft_consensus.cc:2932] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } } }
I20240502 02:04:27.501462 21440 raft_consensus.cc:2932] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } } }
I20240502 02:04:27.501487 21571 raft_consensus.cc:2932] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } } }
I20240502 02:04:27.503499 21292 catalog_manager.cc:5484] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: config changed from index -1 to 4, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "0983e154926c44ceb508483878dba5a0" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } } }
I20240502 02:04:27.511276 21440 consensus_queue.cc:237] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } }
I20240502 02:04:27.512390 21708 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:27.512475 21571 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 02:04:27.512666 21372 consensus_peers.cc:485] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 -> Peer 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69:45619): Couldn't send request to peer 5e9df19de7264d1a9ee930ee770a8701. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6cd9e4d44fdb40448877d5560a58c9f4. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:27.512799 22190 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:27.512964 22940 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:27.514387 22190 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:27.514794 21571 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:27.514986 21708 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:27.517186 21295 catalog_manager.cc:5484] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index -1 to 4, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:27.529911 22495 heartbeater.cc:503] Master 127.7.221.126:38635 was elected leader, sending a full tablet report...
I20240502 02:04:27.704229 22630 heartbeater.cc:503] Master 127.7.221.126:38635 was elected leader, sending a full tablet report...
I20240502 02:04:27.888177 22764 heartbeater.cc:503] Master 127.7.221.126:38635 was elected leader, sending a full tablet report...
I20240502 02:04:27.895776 23060 ts_tablet_manager.cc:902] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701: Initiating tablet copy from peer e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519)
I20240502 02:04:27.896485 23060 tablet_copy_client.cc:308] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:38519
I20240502 02:04:27.899775 21460 tablet_copy_service.cc:140] P e72358c248634f30be5f9becf0ec0fa0: Received BeginTabletCopySession request for tablet c0db5fac24d54971baf964b55e62abe4 from peer 5e9df19de7264d1a9ee930ee770a8701 ({username='slave'} at 127.7.221.69:33381)
I20240502 02:04:27.899907 21460 tablet_copy_service.cc:161] P e72358c248634f30be5f9becf0ec0fa0: Beginning new tablet copy session on tablet c0db5fac24d54971baf964b55e62abe4 from peer 5e9df19de7264d1a9ee930ee770a8701 at {username='slave'} at 127.7.221.69:33381: session id = 5e9df19de7264d1a9ee930ee770a8701-c0db5fac24d54971baf964b55e62abe4
I20240502 02:04:27.900981 21460 tablet_copy_source_session.cc:205] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:27.903981 23060 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c0db5fac24d54971baf964b55e62abe4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:27.908984 23060 tablet_copy_client.cc:783] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:27.909195 23060 tablet_copy_client.cc:647] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:27.910758 23060 tablet_copy_client.cc:523] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:27.912298 23060 tablet_bootstrap.cc:492] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701: Bootstrap starting.
I20240502 02:04:27.917483 23060 log.cc:824] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:27.926441 23067 ts_tablet_manager.cc:902] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701: Initiating tablet copy from peer e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519)
I20240502 02:04:27.926963 23067 tablet_copy_client.cc:308] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:38519
I20240502 02:04:27.927431 21460 tablet_copy_service.cc:140] P e72358c248634f30be5f9becf0ec0fa0: Received BeginTabletCopySession request for tablet 6cd9e4d44fdb40448877d5560a58c9f4 from peer 5e9df19de7264d1a9ee930ee770a8701 ({username='slave'} at 127.7.221.69:33381)
I20240502 02:04:27.927527 21460 tablet_copy_service.cc:161] P e72358c248634f30be5f9becf0ec0fa0: Beginning new tablet copy session on tablet 6cd9e4d44fdb40448877d5560a58c9f4 from peer 5e9df19de7264d1a9ee930ee770a8701 at {username='slave'} at 127.7.221.69:33381: session id = 5e9df19de7264d1a9ee930ee770a8701-6cd9e4d44fdb40448877d5560a58c9f4
I20240502 02:04:27.928206 21460 tablet_copy_source_session.cc:205] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:27.928835 23067 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6cd9e4d44fdb40448877d5560a58c9f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:27.930496 23067 tablet_copy_client.cc:783] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:27.930676 23067 tablet_copy_client.cc:647] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:27.931381 23073 ts_tablet_manager.cc:902] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2: Initiating tablet copy from peer 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:27.931872 23067 tablet_copy_client.cc:523] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:27.932157 23073 tablet_copy_client.cc:308] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:35557
I20240502 02:04:27.933205 23067 tablet_bootstrap.cc:492] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701: Bootstrap starting.
I20240502 02:04:27.936077 21728 tablet_copy_service.cc:140] P 0983e154926c44ceb508483878dba5a0: Received BeginTabletCopySession request for tablet 15a6b979fd044fe9b58f86ffe137b604 from peer d590702730234f799e2a26de695bd7a2 ({username='slave'} at 127.7.221.68:35865)
I20240502 02:04:27.936233 21728 tablet_copy_service.cc:161] P 0983e154926c44ceb508483878dba5a0: Beginning new tablet copy session on tablet 15a6b979fd044fe9b58f86ffe137b604 from peer d590702730234f799e2a26de695bd7a2 at {username='slave'} at 127.7.221.68:35865: session id = d590702730234f799e2a26de695bd7a2-15a6b979fd044fe9b58f86ffe137b604
I20240502 02:04:27.937425 21728 tablet_copy_source_session.cc:205] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:27.939218 23060 tablet_bootstrap.cc:492] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:27.939507 23060 tablet_bootstrap.cc:492] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701: Bootstrap complete.
I20240502 02:04:27.939802 23060 ts_tablet_manager.cc:1367] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701: Time spent bootstrapping tablet: real 0.028s user 0.027s sys 0.000s
I20240502 02:04:27.940325 23073 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 15a6b979fd044fe9b58f86ffe137b604. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:27.942052 23060 raft_consensus.cc:348] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } }
I20240502 02:04:27.942278 23060 raft_consensus.cc:729] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 5e9df19de7264d1a9ee930ee770a8701, State: Initialized, Role: LEARNER
I20240502 02:04:27.942626 23060 consensus_queue.cc:260] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } }
I20240502 02:04:27.943457 23060 ts_tablet_manager.cc:1398] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701: Time spent starting tablet: real 0.004s user 0.004s sys 0.000s
I20240502 02:04:27.945498 21460 tablet_copy_service.cc:342] P e72358c248634f30be5f9becf0ec0fa0: Request end of tablet copy session 5e9df19de7264d1a9ee930ee770a8701-c0db5fac24d54971baf964b55e62abe4 received from {username='slave'} at 127.7.221.69:33381
I20240502 02:04:27.945587 21460 tablet_copy_service.cc:433] P e72358c248634f30be5f9becf0ec0fa0: ending tablet copy session 5e9df19de7264d1a9ee930ee770a8701-c0db5fac24d54971baf964b55e62abe4 on tablet c0db5fac24d54971baf964b55e62abe4 with peer 5e9df19de7264d1a9ee930ee770a8701
I20240502 02:04:27.946275 23073 tablet_copy_client.cc:783] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:27.946465 23073 tablet_copy_client.cc:647] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:27.948580 23073 tablet_copy_client.cc:523] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:27.950079 23073 tablet_bootstrap.cc:492] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2: Bootstrap starting.
I20240502 02:04:27.955610 23073 log.cc:824] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:27.958463 23067 tablet_bootstrap.cc:492] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:27.958734 23067 tablet_bootstrap.cc:492] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701: Bootstrap complete.
I20240502 02:04:27.958882 23067 ts_tablet_manager.cc:1367] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701: Time spent bootstrapping tablet: real 0.026s user 0.022s sys 0.003s
I20240502 02:04:27.959326 23067 raft_consensus.cc:348] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } }
I20240502 02:04:27.959425 23067 raft_consensus.cc:729] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 5e9df19de7264d1a9ee930ee770a8701, State: Initialized, Role: LEARNER
I20240502 02:04:27.959529 23067 consensus_queue.cc:260] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } }
I20240502 02:04:27.959748 23067 ts_tablet_manager.cc:1398] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:27.960806 21460 tablet_copy_service.cc:342] P e72358c248634f30be5f9becf0ec0fa0: Request end of tablet copy session 5e9df19de7264d1a9ee930ee770a8701-6cd9e4d44fdb40448877d5560a58c9f4 received from {username='slave'} at 127.7.221.69:33381
I20240502 02:04:27.960883 21460 tablet_copy_service.cc:433] P e72358c248634f30be5f9becf0ec0fa0: ending tablet copy session 5e9df19de7264d1a9ee930ee770a8701-6cd9e4d44fdb40448877d5560a58c9f4 on tablet 6cd9e4d44fdb40448877d5560a58c9f4 with peer 5e9df19de7264d1a9ee930ee770a8701
I20240502 02:04:27.974169 23082 ts_tablet_manager.cc:902] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc: Initiating tablet copy from peer 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219)
I20240502 02:04:27.974977 23082 tablet_copy_client.cc:308] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34219
I20240502 02:04:27.976925 23073 tablet_bootstrap.cc:492] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:27.977272 23073 tablet_bootstrap.cc:492] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2: Bootstrap complete.
I20240502 02:04:27.977600 23073 ts_tablet_manager.cc:1367] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2: Time spent bootstrapping tablet: real 0.028s user 0.021s sys 0.004s
I20240502 02:04:27.979413 21595 tablet_copy_service.cc:140] P 252575ab96ac4a079bab6ade63792782: Received BeginTabletCopySession request for tablet 1683e87385ab4050824a71508d3c3f5b from peer 2590825fe67b48c9adb5894c322237dc ({username='slave'} at 127.7.221.70:52301)
I20240502 02:04:27.979564 21595 tablet_copy_service.cc:161] P 252575ab96ac4a079bab6ade63792782: Beginning new tablet copy session on tablet 1683e87385ab4050824a71508d3c3f5b from peer 2590825fe67b48c9adb5894c322237dc at {username='slave'} at 127.7.221.70:52301: session id = 2590825fe67b48c9adb5894c322237dc-1683e87385ab4050824a71508d3c3f5b
I20240502 02:04:27.980022 23073 raft_consensus.cc:348] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } }
I20240502 02:04:27.980257 23073 raft_consensus.cc:729] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d590702730234f799e2a26de695bd7a2, State: Initialized, Role: LEARNER
I20240502 02:04:27.980577 23073 consensus_queue.cc:260] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } }
I20240502 02:04:27.980928 21595 tablet_copy_source_session.cc:205] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:27.981408 23073 ts_tablet_manager.cc:1398] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2: Time spent starting tablet: real 0.004s user 0.006s sys 0.000s
I20240502 02:04:27.982690 21728 tablet_copy_service.cc:342] P 0983e154926c44ceb508483878dba5a0: Request end of tablet copy session d590702730234f799e2a26de695bd7a2-15a6b979fd044fe9b58f86ffe137b604 received from {username='slave'} at 127.7.221.68:35865
I20240502 02:04:27.982791 21728 tablet_copy_service.cc:433] P 0983e154926c44ceb508483878dba5a0: ending tablet copy session d590702730234f799e2a26de695bd7a2-15a6b979fd044fe9b58f86ffe137b604 on tablet 15a6b979fd044fe9b58f86ffe137b604 with peer d590702730234f799e2a26de695bd7a2
I20240502 02:04:27.986059 23082 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1683e87385ab4050824a71508d3c3f5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:27.986434 23073 ts_tablet_manager.cc:902] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2: Initiating tablet copy from peer 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:27.986868 23073 tablet_copy_client.cc:308] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:35557
I20240502 02:04:27.987284 21728 tablet_copy_service.cc:140] P 0983e154926c44ceb508483878dba5a0: Received BeginTabletCopySession request for tablet fd1b849d9ecd4f5e96b678673444f4d6 from peer d590702730234f799e2a26de695bd7a2 ({username='slave'} at 127.7.221.68:35865)
I20240502 02:04:27.987404 21728 tablet_copy_service.cc:161] P 0983e154926c44ceb508483878dba5a0: Beginning new tablet copy session on tablet fd1b849d9ecd4f5e96b678673444f4d6 from peer d590702730234f799e2a26de695bd7a2 at {username='slave'} at 127.7.221.68:35865: session id = d590702730234f799e2a26de695bd7a2-fd1b849d9ecd4f5e96b678673444f4d6
I20240502 02:04:27.988332 21728 tablet_copy_source_session.cc:205] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:27.989041 23073 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fd1b849d9ecd4f5e96b678673444f4d6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:27.990653 23073 tablet_copy_client.cc:783] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:27.990839 23073 tablet_copy_client.cc:647] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:27.991969 23073 tablet_copy_client.cc:523] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:27.992158 23082 tablet_copy_client.cc:783] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:27.992336 23082 tablet_copy_client.cc:647] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:27.993768 23073 tablet_bootstrap.cc:492] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2: Bootstrap starting.
I20240502 02:04:27.994119 23082 tablet_copy_client.cc:523] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:27.995725 23082 tablet_bootstrap.cc:492] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc: Bootstrap starting.
I20240502 02:04:28.002238 23082 log.cc:824] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:28.018138 23073 tablet_bootstrap.cc:492] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:28.018464 23073 tablet_bootstrap.cc:492] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2: Bootstrap complete.
I20240502 02:04:28.018687 23073 ts_tablet_manager.cc:1367] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2: Time spent bootstrapping tablet: real 0.025s user 0.022s sys 0.003s
I20240502 02:04:28.019451 23073 raft_consensus.cc:348] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } }
I20240502 02:04:28.019596 23073 raft_consensus.cc:729] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d590702730234f799e2a26de695bd7a2, State: Initialized, Role: LEARNER
I20240502 02:04:28.019711 23073 consensus_queue.cc:260] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } }
I20240502 02:04:28.020545 23073 ts_tablet_manager.cc:1398] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 02:04:28.021001 21728 tablet_copy_service.cc:342] P 0983e154926c44ceb508483878dba5a0: Request end of tablet copy session d590702730234f799e2a26de695bd7a2-fd1b849d9ecd4f5e96b678673444f4d6 received from {username='slave'} at 127.7.221.68:35865
I20240502 02:04:28.021078 21728 tablet_copy_service.cc:433] P 0983e154926c44ceb508483878dba5a0: ending tablet copy session d590702730234f799e2a26de695bd7a2-fd1b849d9ecd4f5e96b678673444f4d6 on tablet fd1b849d9ecd4f5e96b678673444f4d6 with peer d590702730234f799e2a26de695bd7a2
I20240502 02:04:28.024720 23067 ts_tablet_manager.cc:902] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701: Initiating tablet copy from peer e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519)
I20240502 02:04:28.025194 23067 tablet_copy_client.cc:308] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:38519
I20240502 02:04:28.025709 21460 tablet_copy_service.cc:140] P e72358c248634f30be5f9becf0ec0fa0: Received BeginTabletCopySession request for tablet a22071eab1054f0ebc97dfd4e0515148 from peer 5e9df19de7264d1a9ee930ee770a8701 ({username='slave'} at 127.7.221.69:33381)
I20240502 02:04:28.025812 21460 tablet_copy_service.cc:161] P e72358c248634f30be5f9becf0ec0fa0: Beginning new tablet copy session on tablet a22071eab1054f0ebc97dfd4e0515148 from peer 5e9df19de7264d1a9ee930ee770a8701 at {username='slave'} at 127.7.221.69:33381: session id = 5e9df19de7264d1a9ee930ee770a8701-a22071eab1054f0ebc97dfd4e0515148
I20240502 02:04:28.026563 21460 tablet_copy_source_session.cc:205] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:28.027529 23067 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a22071eab1054f0ebc97dfd4e0515148. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:28.030010 23067 tablet_copy_client.cc:783] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:28.030170 23067 tablet_copy_client.cc:647] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:28.031420 23067 tablet_copy_client.cc:523] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:28.031744 23082 tablet_bootstrap.cc:492] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:28.032079 23082 tablet_bootstrap.cc:492] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc: Bootstrap complete.
I20240502 02:04:28.032374 23082 ts_tablet_manager.cc:1367] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc: Time spent bootstrapping tablet: real 0.037s user 0.031s sys 0.005s
I20240502 02:04:28.032487 23067 tablet_bootstrap.cc:492] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701: Bootstrap starting.
I20240502 02:04:28.034420 23082 raft_consensus.cc:348] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } }
I20240502 02:04:28.034641 23082 raft_consensus.cc:729] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 2590825fe67b48c9adb5894c322237dc, State: Initialized, Role: LEARNER
I20240502 02:04:28.034933 23082 consensus_queue.cc:260] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } }
I20240502 02:04:28.035663 23082 ts_tablet_manager.cc:1398] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc: Time spent starting tablet: real 0.003s user 0.000s sys 0.004s
I20240502 02:04:28.036406 21595 tablet_copy_service.cc:342] P 252575ab96ac4a079bab6ade63792782: Request end of tablet copy session 2590825fe67b48c9adb5894c322237dc-1683e87385ab4050824a71508d3c3f5b received from {username='slave'} at 127.7.221.70:52301
I20240502 02:04:28.036515 21595 tablet_copy_service.cc:433] P 252575ab96ac4a079bab6ade63792782: ending tablet copy session 2590825fe67b48c9adb5894c322237dc-1683e87385ab4050824a71508d3c3f5b on tablet 1683e87385ab4050824a71508d3c3f5b with peer 2590825fe67b48c9adb5894c322237dc
I20240502 02:04:28.053279 23067 tablet_bootstrap.cc:492] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:28.053561 23067 tablet_bootstrap.cc:492] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701: Bootstrap complete.
I20240502 02:04:28.053719 23067 ts_tablet_manager.cc:1367] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701: Time spent bootstrapping tablet: real 0.021s user 0.019s sys 0.000s
I20240502 02:04:28.054087 23067 raft_consensus.cc:348] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } }
I20240502 02:04:28.054200 23067 raft_consensus.cc:729] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 5e9df19de7264d1a9ee930ee770a8701, State: Initialized, Role: LEARNER
I20240502 02:04:28.054301 23067 consensus_queue.cc:260] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } }
I20240502 02:04:28.054519 23067 ts_tablet_manager.cc:1398] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:28.055011 21460 tablet_copy_service.cc:342] P e72358c248634f30be5f9becf0ec0fa0: Request end of tablet copy session 5e9df19de7264d1a9ee930ee770a8701-a22071eab1054f0ebc97dfd4e0515148 received from {username='slave'} at 127.7.221.69:33381
I20240502 02:04:28.055092 21460 tablet_copy_service.cc:433] P e72358c248634f30be5f9becf0ec0fa0: ending tablet copy session 5e9df19de7264d1a9ee930ee770a8701-a22071eab1054f0ebc97dfd4e0515148 on tablet a22071eab1054f0ebc97dfd4e0515148 with peer 5e9df19de7264d1a9ee930ee770a8701
I20240502 02:04:28.081879 23082 ts_tablet_manager.cc:902] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc: Initiating tablet copy from peer e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519)
I20240502 02:04:28.082355 23082 tablet_copy_client.cc:308] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:38519
I20240502 02:04:28.085817 21460 tablet_copy_service.cc:140] P e72358c248634f30be5f9becf0ec0fa0: Received BeginTabletCopySession request for tablet fe41f0bd0a884eaa9def3489cece439f from peer 2590825fe67b48c9adb5894c322237dc ({username='slave'} at 127.7.221.70:35727)
I20240502 02:04:28.085922 21460 tablet_copy_service.cc:161] P e72358c248634f30be5f9becf0ec0fa0: Beginning new tablet copy session on tablet fe41f0bd0a884eaa9def3489cece439f from peer 2590825fe67b48c9adb5894c322237dc at {username='slave'} at 127.7.221.70:35727: session id = 2590825fe67b48c9adb5894c322237dc-fe41f0bd0a884eaa9def3489cece439f
I20240502 02:04:28.086675 21460 tablet_copy_source_session.cc:205] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:28.087493 23082 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fe41f0bd0a884eaa9def3489cece439f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:28.089151 23082 tablet_copy_client.cc:783] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:28.089399 23082 tablet_copy_client.cc:647] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:28.090420 23082 tablet_copy_client.cc:523] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:28.091533 23082 tablet_bootstrap.cc:492] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc: Bootstrap starting.
I20240502 02:04:28.107710 23082 tablet_bootstrap.cc:492] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:28.107998 23082 tablet_bootstrap.cc:492] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc: Bootstrap complete.
I20240502 02:04:28.108181 23082 ts_tablet_manager.cc:1367] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc: Time spent bootstrapping tablet: real 0.017s user 0.012s sys 0.004s
I20240502 02:04:28.108765 23082 raft_consensus.cc:348] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } }
I20240502 02:04:28.108893 23082 raft_consensus.cc:729] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 2590825fe67b48c9adb5894c322237dc, State: Initialized, Role: LEARNER
I20240502 02:04:28.108971 23082 consensus_queue.cc:260] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } }
I20240502 02:04:28.109184 23082 ts_tablet_manager.cc:1398] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:28.109633 21460 tablet_copy_service.cc:342] P e72358c248634f30be5f9becf0ec0fa0: Request end of tablet copy session 2590825fe67b48c9adb5894c322237dc-fe41f0bd0a884eaa9def3489cece439f received from {username='slave'} at 127.7.221.70:35727
I20240502 02:04:28.109721 21460 tablet_copy_service.cc:433] P e72358c248634f30be5f9becf0ec0fa0: ending tablet copy session 2590825fe67b48c9adb5894c322237dc-fe41f0bd0a884eaa9def3489cece439f on tablet fe41f0bd0a884eaa9def3489cece439f with peer 2590825fe67b48c9adb5894c322237dc
I20240502 02:04:28.389137 22449 raft_consensus.cc:1203] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 02:04:28.400451 22718 raft_consensus.cc:1203] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 02:04:28.446146 22449 raft_consensus.cc:1203] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 02:04:28.481395 22718 raft_consensus.cc:1203] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 02:04:28.486088 22582 raft_consensus.cc:1203] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 02:04:28.500882 22449 raft_consensus.cc:1203] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 02:04:28.571704 22582 raft_consensus.cc:1203] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 02:04:28.825194 23182 raft_consensus.cc:1050] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0: attempting to promote NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 to VOTER
I20240502 02:04:28.825655 23182 consensus_queue.cc:237] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } }
I20240502 02:04:28.827111 21708 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:28.827126 22449 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:28.827154 21571 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:28.827607 23259 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:28.827802 23182 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:28.827940 22939 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:28.830793 23182 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.832305 21708 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.832544 21571 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.834683 21295 catalog_manager.cc:5484] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 reported cstate change: config changed from index 4 to 5, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.835671 22449 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.839824 21440 consensus_queue.cc:237] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } }
I20240502 02:04:28.840772 21708 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:28.841043 22449 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:28.841078 23182 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:28.841400 22939 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:28.842491 23182 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.842964 21708 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.843762 21282 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6cd9e4d44fdb40448877d5560a58c9f4 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:28.844527 22449 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.845019 21294 catalog_manager.cc:5484] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 reported cstate change: config changed from index 5 to 6, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.847555 21551 tablet_service.cc:1514] Processing DeleteTablet for tablet 6cd9e4d44fdb40448877d5560a58c9f4 with delete_type TABLET_DATA_TOMBSTONED (TS 252575ab96ac4a079bab6ade63792782 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:43552
I20240502 02:04:28.848047 23267 tablet_replica.cc:330] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782: stopping tablet replica
I20240502 02:04:28.848377 23267 raft_consensus.cc:2229] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:28.848583 23267 raft_consensus.cc:2258] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:28.849622 23267 ts_tablet_manager.cc:1875] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:28.851429 23267 ts_tablet_manager.cc:1888] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 02:04:28.851508 23267 log.cc:1196] T 6cd9e4d44fdb40448877d5560a58c9f4 P 252575ab96ac4a079bab6ade63792782: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/6cd9e4d44fdb40448877d5560a58c9f4
I20240502 02:04:28.852097 21280 catalog_manager.cc:4830] TS 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219): tablet 6cd9e4d44fdb40448877d5560a58c9f4 (table rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]) successfully deleted
I20240502 02:04:28.913723 23259 raft_consensus.cc:1050] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0: attempting to promote NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 to VOTER
I20240502 02:04:28.914220 23259 consensus_queue.cc:237] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } }
I20240502 02:04:28.915771 21571 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:28.915797 21708 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:28.915884 22449 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:28.916224 22939 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:28.916457 23182 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:28.916689 23138 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:28.918661 22939 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.919173 22449 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.919545 21571 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.920764 21708 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.921766 21292 catalog_manager.cc:5484] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 reported cstate change: config changed from index 4 to 5, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.925521 21440 consensus_queue.cc:237] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } }
I20240502 02:04:28.926561 21708 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:28.926695 22449 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:28.926988 23182 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:28.927343 23138 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:28.928467 23182 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.929133 22449 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.929787 21282 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c0db5fac24d54971baf964b55e62abe4 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:28.930294 21708 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.931005 21290 catalog_manager.cc:5484] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: config changed from index 5 to 6, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:28.933754 21551 tablet_service.cc:1514] Processing DeleteTablet for tablet c0db5fac24d54971baf964b55e62abe4 with delete_type TABLET_DATA_TOMBSTONED (TS 252575ab96ac4a079bab6ade63792782 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:43552
I20240502 02:04:28.933996 23267 tablet_replica.cc:330] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: stopping tablet replica
I20240502 02:04:28.934113 23267 raft_consensus.cc:2229] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:28.934257 23267 raft_consensus.cc:2258] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:28.934872 23267 ts_tablet_manager.cc:1875] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:28.936393 23267 ts_tablet_manager.cc:1888] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 02:04:28.936458 23267 log.cc:1196] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/c0db5fac24d54971baf964b55e62abe4
I20240502 02:04:28.936918 21280 catalog_manager.cc:4830] TS 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219): tablet c0db5fac24d54971baf964b55e62abe4 (table rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]) successfully deleted
I20240502 02:04:28.978039 23259 raft_consensus.cc:1050] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0: attempting to promote NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 to VOTER
I20240502 02:04:28.978448 23259 consensus_queue.cc:237] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } }
I20240502 02:04:28.979868 21708 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:28.979871 22449 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:28.979893 21571 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:28.980283 22939 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:28.980481 23259 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:28.980651 23138 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:28.984156 23138 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.984632 21708 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.986737 22449 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.987076 21294 catalog_manager.cc:5484] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 reported cstate change: config changed from index 4 to 5, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.990556 21440 consensus_queue.cc:237] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } }
I20240502 02:04:28.991789 21708 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:28.991837 23181 raft_consensus.cc:1050] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0: attempting to promote NON_VOTER d590702730234f799e2a26de695bd7a2 to VOTER
I20240502 02:04:28.992157 23259 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:28.992296 23181 consensus_queue.cc:237] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } }
I20240502 02:04:28.993774 22718 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:28.993819 22449 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:28.994001 21573 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:28.994400 23306 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:28.994334 21571 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.994412 23138 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.995836 21708 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.996714 21440 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:28.997606 23182 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:28.997793 21282 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a22071eab1054f0ebc97dfd4e0515148 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:28.998521 21295 catalog_manager.cc:5484] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 reported cstate change: config changed from index 5 to 6, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:28.999128 22449 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:29.000213 23308 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.002s
I20240502 02:04:29.001948 21551 tablet_service.cc:1514] Processing DeleteTablet for tablet a22071eab1054f0ebc97dfd4e0515148 with delete_type TABLET_DATA_TOMBSTONED (TS 252575ab96ac4a079bab6ade63792782 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:43552
I20240502 02:04:29.002127 23267 tablet_replica.cc:330] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: stopping tablet replica
I20240502 02:04:29.002277 23267 raft_consensus.cc:2229] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:29.002429 23267 raft_consensus.cc:2258] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:29.002699 23306 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:29.003099 23267 ts_tablet_manager.cc:1875] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:29.004021 23301 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:29.004433 21439 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:29.006295 21573 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:29.006561 21290 catalog_manager.cc:5484] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 reported cstate change: config changed from index 4 to 5, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "0983e154926c44ceb508483878dba5a0" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:29.007227 23267 ts_tablet_manager.cc:1888] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 02:04:29.007311 23267 log.cc:1196] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/a22071eab1054f0ebc97dfd4e0515148
I20240502 02:04:29.008669 21280 catalog_manager.cc:4830] TS 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219): tablet a22071eab1054f0ebc97dfd4e0515148 (table rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]) successfully deleted
I20240502 02:04:29.008850 22718 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:29.012849 21708 consensus_queue.cc:237] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } }
I20240502 02:04:29.014081 21573 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:29.014273 22718 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:29.014618 23308 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:29.014981 23306 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:29.016253 23308 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:29.016733 22718 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:29.017609 21283 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet fd1b849d9ecd4f5e96b678673444f4d6 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:29.018512 21573 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:29.019039 21295 catalog_manager.cc:5484] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 reported cstate change: config changed from index 5 to 6, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "0983e154926c44ceb508483878dba5a0" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:29.021687 21417 tablet_service.cc:1514] Processing DeleteTablet for tablet fd1b849d9ecd4f5e96b678673444f4d6 with delete_type TABLET_DATA_TOMBSTONED (TS e72358c248634f30be5f9becf0ec0fa0 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:56444
I20240502 02:04:29.022214 23324 tablet_replica.cc:330] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: stopping tablet replica
I20240502 02:04:29.022534 23324 raft_consensus.cc:2229] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:29.022778 23324 raft_consensus.cc:2258] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:29.023463 23308 raft_consensus.cc:1050] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0: attempting to promote NON_VOTER d590702730234f799e2a26de695bd7a2 to VOTER
I20240502 02:04:29.023861 23308 consensus_queue.cc:237] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } }
I20240502 02:04:29.024119 23324 ts_tablet_manager.cc:1875] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:29.025342 22718 raft_consensus.cc:1261] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:29.025681 23181 consensus_queue.cc:1035] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:29.026360 21573 raft_consensus.cc:1261] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:29.026749 23181 consensus_queue.cc:1035] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:29.026779 23324 ts_tablet_manager.cc:1888] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 02:04:29.026866 23324 log.cc:1196] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/fd1b849d9ecd4f5e96b678673444f4d6
I20240502 02:04:29.027459 21282 catalog_manager.cc:4830] TS e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519): tablet fd1b849d9ecd4f5e96b678673444f4d6 (table rebalance_test_table_2 [id=c5c742d762c44789b376f2c4c64c13b2]) successfully deleted
I20240502 02:04:29.028004 21439 raft_consensus.cc:1261] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:29.028365 23308 consensus_queue.cc:1035] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:29.028517 23306 raft_consensus.cc:2932] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:29.029943 21439 raft_consensus.cc:2932] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:29.031647 22718 raft_consensus.cc:2932] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:29.031916 21290 catalog_manager.cc:5484] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 reported cstate change: config changed from index 4 to 5, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "0983e154926c44ceb508483878dba5a0" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:29.032724 21573 raft_consensus.cc:2932] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:29.037906 21708 consensus_queue.cc:237] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } }
I20240502 02:04:29.038929 21573 raft_consensus.cc:1261] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:29.039011 22718 raft_consensus.cc:1261] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:29.039376 23335 consensus_queue.cc:1035] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:29.039588 23335 consensus_queue.cc:1035] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:29.040975 23335 raft_consensus.cc:2932] T 15a6b979fd044fe9b58f86ffe137b604 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:29.041375 21573 raft_consensus.cc:2932] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:29.042917 22718 raft_consensus.cc:2932] T 15a6b979fd044fe9b58f86ffe137b604 P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:29.043617 21294 catalog_manager.cc:5484] T 15a6b979fd044fe9b58f86ffe137b604 P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index 5 to 6, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "0983e154926c44ceb508483878dba5a0" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:29.044348 21283 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 15a6b979fd044fe9b58f86ffe137b604 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:29.044523 23138 raft_consensus.cc:1050] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0: attempting to promote NON_VOTER 2590825fe67b48c9adb5894c322237dc to VOTER
I20240502 02:04:29.044965 23138 consensus_queue.cc:237] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } }
I20240502 02:04:29.046413 22582 raft_consensus.cc:1261] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:29.047003 21708 raft_consensus.cc:1261] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:29.047390 23259 consensus_queue.cc:1035] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:29.047551 21417 tablet_service.cc:1514] Processing DeleteTablet for tablet 15a6b979fd044fe9b58f86ffe137b604 with delete_type TABLET_DATA_TOMBSTONED (TS e72358c248634f30be5f9becf0ec0fa0 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:56444
I20240502 02:04:29.047693 23182 consensus_queue.cc:1035] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:29.048101 23324 tablet_replica.cc:330] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0: stopping tablet replica
I20240502 02:04:29.048257 23324 raft_consensus.cc:2229] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:29.048391 23324 raft_consensus.cc:2258] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:29.048444 21573 raft_consensus.cc:1261] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:29.048835 23138 consensus_queue.cc:1035] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:29.049404 23324 ts_tablet_manager.cc:1875] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:29.051723 23324 ts_tablet_manager.cc:1888] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 02:04:29.051803 23324 log.cc:1196] T 15a6b979fd044fe9b58f86ffe137b604 P e72358c248634f30be5f9becf0ec0fa0: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/15a6b979fd044fe9b58f86ffe137b604
I20240502 02:04:29.052137 23182 raft_consensus.cc:2932] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:29.052326 21282 catalog_manager.cc:4830] TS e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519): tablet 15a6b979fd044fe9b58f86ffe137b604 (table rebalance_test_table_0 [id=e1df1cf914834ab4bf5ac68bc944cbcc]) successfully deleted
I20240502 02:04:29.052354 21573 raft_consensus.cc:2932] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:29.054195 22582 raft_consensus.cc:2932] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:29.054553 21295 catalog_manager.cc:5484] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: config changed from index 4 to 5, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:29.056524 21708 raft_consensus.cc:2932] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:29.057708 21439 consensus_queue.cc:237] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } }
I20240502 02:04:29.059036 21573 raft_consensus.cc:1261] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:29.059355 22582 raft_consensus.cc:1261] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:29.059363 22939 consensus_queue.cc:1035] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:29.060252 22939 consensus_queue.cc:1035] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:29.061043 23259 raft_consensus.cc:2932] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:29.061442 21573 raft_consensus.cc:2932] T fe41f0bd0a884eaa9def3489cece439f P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:29.063280 21282 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet fe41f0bd0a884eaa9def3489cece439f with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:29.063227 21295 catalog_manager.cc:5484] T fe41f0bd0a884eaa9def3489cece439f P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: config changed from index 5 to 6, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:29.064244 22582 raft_consensus.cc:2932] T fe41f0bd0a884eaa9def3489cece439f P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:29.066102 21687 tablet_service.cc:1514] Processing DeleteTablet for tablet fe41f0bd0a884eaa9def3489cece439f with delete_type TABLET_DATA_TOMBSTONED (TS 0983e154926c44ceb508483878dba5a0 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:45734
I20240502 02:04:29.066653 23355 tablet_replica.cc:330] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0: stopping tablet replica
I20240502 02:04:29.066991 23355 raft_consensus.cc:2229] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:29.067237 23355 raft_consensus.cc:2258] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:29.068516 23355 ts_tablet_manager.cc:1875] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:29.070267 23355 ts_tablet_manager.cc:1888] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 02:04:29.070343 23355 log.cc:1196] T fe41f0bd0a884eaa9def3489cece439f P 0983e154926c44ceb508483878dba5a0: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/fe41f0bd0a884eaa9def3489cece439f
I20240502 02:04:29.070868 21283 catalog_manager.cc:4830] TS 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557): tablet fe41f0bd0a884eaa9def3489cece439f (table rebalance_test_table_2 [id=c5c742d762c44789b376f2c4c64c13b2]) successfully deleted
I20240502 02:04:29.145475 23371 raft_consensus.cc:1050] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782: attempting to promote NON_VOTER 2590825fe67b48c9adb5894c322237dc to VOTER
I20240502 02:04:29.145957 23371 consensus_queue.cc:237] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } }
I20240502 02:04:29.147370 22582 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:29.147413 21439 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:29.147439 21708 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:29.147890 23373 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:29.148097 23374 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:29.148305 23371 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:29.150141 23373 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:29.150521 22582 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:29.150626 21439 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:29.152560 21708 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:29.153160 21292 catalog_manager.cc:5484] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: config changed from index 4 to 5, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "252575ab96ac4a079bab6ade63792782" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:29.156463 21573 consensus_queue.cc:237] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } }
I20240502 02:04:29.157517 22582 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:29.157521 21439 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:29.157900 23374 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:29.158078 23220 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:29.159463 23371 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:29.159883 21439 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:29.159893 22582 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:29.160933 21280 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1683e87385ab4050824a71508d3c3f5b with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:29.162127 21294 catalog_manager.cc:5484] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index 5 to 6, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "252575ab96ac4a079bab6ade63792782" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:29.164405 21687 tablet_service.cc:1514] Processing DeleteTablet for tablet 1683e87385ab4050824a71508d3c3f5b with delete_type TABLET_DATA_TOMBSTONED (TS 0983e154926c44ceb508483878dba5a0 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:45734
I20240502 02:04:29.164634 23355 tablet_replica.cc:330] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0: stopping tablet replica
I20240502 02:04:29.164793 23355 raft_consensus.cc:2229] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:29.164952 23355 raft_consensus.cc:2258] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:29.165591 23355 ts_tablet_manager.cc:1875] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:29.167290 23355 ts_tablet_manager.cc:1888] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 02:04:29.167356 23355 log.cc:1196] T 1683e87385ab4050824a71508d3c3f5b P 0983e154926c44ceb508483878dba5a0: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/1683e87385ab4050824a71508d3c3f5b
I20240502 02:04:29.167833 21283 catalog_manager.cc:4830] TS 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557): tablet 1683e87385ab4050824a71508d3c3f5b (table rebalance_test_table_0 [id=e1df1cf914834ab4bf5ac68bc944cbcc]) successfully deleted
I20240502 02:04:29.337529 21687 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:29.338874 21551 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:29.340641 22562 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:29.344985 21417 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 02:04:29.351176 22428 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:29.366820 22697 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:29.392338 21708 consensus_queue.cc:237] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } }
I20240502 02:04:29.393810 21439 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:29.394037 21573 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:29.394374 23335 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:29.394551 23306 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:29.396380 23308 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } } }
I20240502 02:04:29.396816 21573 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } } }
I20240502 02:04:29.396989 21439 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } } }
I20240502 02:04:29.399787 21294 catalog_manager.cc:5484] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: config changed from index -1 to 4, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "0983e154926c44ceb508483878dba5a0" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } } }
W20240502 02:04:29.400031 21641 consensus_peers.cc:485] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 -> Peer 2590825fe67b48c9adb5894c322237dc (127.7.221.70:39691): Couldn't send request to peer 2590825fe67b48c9adb5894c322237dc. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2fd616a594f245a3b13353f1c9cb9064. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:29.406949 21439 consensus_queue.cc:237] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } }
I20240502 02:04:29.408339 21573 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:29.408362 21708 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:29.408713 23138 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:29.409157 23138 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:29.410934 23259 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } } }
I20240502 02:04:29.411379 21708 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } } }
I20240502 02:04:29.411366 21573 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } } }
W20240502 02:04:29.413359 21374 consensus_peers.cc:485] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 -> Peer d590702730234f799e2a26de695bd7a2 (127.7.221.68:46455): Couldn't send request to peer d590702730234f799e2a26de695bd7a2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bb5988c6c4c14f46a5e598cc1a59c99b. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:29.413703 21290 catalog_manager.cc:5484] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index -1 to 4, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } } }
I20240502 02:04:29.802327 23456 ts_tablet_manager.cc:902] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc: Initiating tablet copy from peer 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:29.802874 23456 tablet_copy_client.cc:308] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:35557
I20240502 02:04:29.806017 21728 tablet_copy_service.cc:140] P 0983e154926c44ceb508483878dba5a0: Received BeginTabletCopySession request for tablet 2fd616a594f245a3b13353f1c9cb9064 from peer 2590825fe67b48c9adb5894c322237dc ({username='slave'} at 127.7.221.70:37763)
I20240502 02:04:29.806105 21728 tablet_copy_service.cc:161] P 0983e154926c44ceb508483878dba5a0: Beginning new tablet copy session on tablet 2fd616a594f245a3b13353f1c9cb9064 from peer 2590825fe67b48c9adb5894c322237dc at {username='slave'} at 127.7.221.70:37763: session id = 2590825fe67b48c9adb5894c322237dc-2fd616a594f245a3b13353f1c9cb9064
I20240502 02:04:29.806809 21728 tablet_copy_source_session.cc:205] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:29.807622 23456 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2fd616a594f245a3b13353f1c9cb9064. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:29.809288 23456 tablet_copy_client.cc:783] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:29.809437 23456 tablet_copy_client.cc:647] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:29.810392 23456 tablet_copy_client.cc:523] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:29.811331 23456 tablet_bootstrap.cc:492] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc: Bootstrap starting.
I20240502 02:04:29.827730 23456 tablet_bootstrap.cc:492] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:29.827952 23456 tablet_bootstrap.cc:492] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc: Bootstrap complete.
I20240502 02:04:29.828080 23456 ts_tablet_manager.cc:1367] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc: Time spent bootstrapping tablet: real 0.017s user 0.014s sys 0.002s
I20240502 02:04:29.828423 23456 raft_consensus.cc:348] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } }
I20240502 02:04:29.828521 23456 raft_consensus.cc:729] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 2590825fe67b48c9adb5894c322237dc, State: Initialized, Role: LEARNER
I20240502 02:04:29.828593 23456 consensus_queue.cc:260] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } }
I20240502 02:04:29.828884 23456 ts_tablet_manager.cc:1398] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 02:04:29.829499 21728 tablet_copy_service.cc:342] P 0983e154926c44ceb508483878dba5a0: Request end of tablet copy session 2590825fe67b48c9adb5894c322237dc-2fd616a594f245a3b13353f1c9cb9064 received from {username='slave'} at 127.7.221.70:37763
I20240502 02:04:29.829573 21728 tablet_copy_service.cc:433] P 0983e154926c44ceb508483878dba5a0: ending tablet copy session 2590825fe67b48c9adb5894c322237dc-2fd616a594f245a3b13353f1c9cb9064 on tablet 2fd616a594f245a3b13353f1c9cb9064 with peer 2590825fe67b48c9adb5894c322237dc
I20240502 02:04:29.996706 23475 ts_tablet_manager.cc:902] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: Initiating tablet copy from peer e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519)
I20240502 02:04:29.997287 23475 tablet_copy_client.cc:308] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:38519
I20240502 02:04:30.000277 21460 tablet_copy_service.cc:140] P e72358c248634f30be5f9becf0ec0fa0: Received BeginTabletCopySession request for tablet bb5988c6c4c14f46a5e598cc1a59c99b from peer d590702730234f799e2a26de695bd7a2 ({username='slave'} at 127.7.221.68:36261)
I20240502 02:04:30.000396 21460 tablet_copy_service.cc:161] P e72358c248634f30be5f9becf0ec0fa0: Beginning new tablet copy session on tablet bb5988c6c4c14f46a5e598cc1a59c99b from peer d590702730234f799e2a26de695bd7a2 at {username='slave'} at 127.7.221.68:36261: session id = d590702730234f799e2a26de695bd7a2-bb5988c6c4c14f46a5e598cc1a59c99b
I20240502 02:04:30.001118 21460 tablet_copy_source_session.cc:205] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:30.001907 23475 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bb5988c6c4c14f46a5e598cc1a59c99b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:30.003991 23475 tablet_copy_client.cc:783] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:30.004127 23475 tablet_copy_client.cc:647] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:30.005224 23475 tablet_copy_client.cc:523] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:30.006134 23475 tablet_bootstrap.cc:492] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: Bootstrap starting.
I20240502 02:04:30.022202 23475 tablet_bootstrap.cc:492] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:30.022492 23475 tablet_bootstrap.cc:492] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: Bootstrap complete.
I20240502 02:04:30.022696 23475 ts_tablet_manager.cc:1367] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: Time spent bootstrapping tablet: real 0.017s user 0.010s sys 0.007s
I20240502 02:04:30.023072 23475 raft_consensus.cc:348] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } }
I20240502 02:04:30.023180 23475 raft_consensus.cc:729] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d590702730234f799e2a26de695bd7a2, State: Initialized, Role: LEARNER
I20240502 02:04:30.023268 23475 consensus_queue.cc:260] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } }
I20240502 02:04:30.023557 23475 ts_tablet_manager.cc:1398] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:30.024056 21460 tablet_copy_service.cc:342] P e72358c248634f30be5f9becf0ec0fa0: Request end of tablet copy session d590702730234f799e2a26de695bd7a2-bb5988c6c4c14f46a5e598cc1a59c99b received from {username='slave'} at 127.7.221.68:36261
I20240502 02:04:30.024137 21460 tablet_copy_service.cc:433] P e72358c248634f30be5f9becf0ec0fa0: ending tablet copy session d590702730234f799e2a26de695bd7a2-bb5988c6c4c14f46a5e598cc1a59c99b on tablet bb5988c6c4c14f46a5e598cc1a59c99b with peer d590702730234f799e2a26de695bd7a2
I20240502 02:04:30.391328 22582 raft_consensus.cc:1203] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 02:04:30.553490 22718 raft_consensus.cc:1203] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 02:04:30.831724 23306 raft_consensus.cc:1050] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: attempting to promote NON_VOTER 2590825fe67b48c9adb5894c322237dc to VOTER
I20240502 02:04:30.832099 23306 consensus_queue.cc:237] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } }
I20240502 02:04:30.833386 22582 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:30.833415 21439 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:30.833554 21573 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:30.833782 23532 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:30.833956 23506 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:30.834100 23306 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:30.835911 23506 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:30.836318 21573 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:30.836483 22582 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:30.838011 21439 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:30.839114 21290 catalog_manager.cc:5484] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index 4 to 5, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "0983e154926c44ceb508483878dba5a0" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:30.915041 21708 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "2fd616a594f245a3b13353f1c9cb9064"
dest_uuid: "0983e154926c44ceb508483878dba5a0"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:38048
I20240502 02:04:30.915208 21708 raft_consensus.cc:595] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:31.007575 23138 raft_consensus.cc:1050] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0: attempting to promote NON_VOTER d590702730234f799e2a26de695bd7a2 to VOTER
I20240502 02:04:31.007961 23138 consensus_queue.cc:237] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } }
I20240502 02:04:31.009078 22718 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:31.009222 21708 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:31.009272 21573 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:31.009476 23557 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:31.009696 23474 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:31.009869 23505 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:31.011751 23474 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:31.012159 21573 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:31.012212 22718 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:31.013936 21708 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:31.014920 21295 catalog_manager.cc:5484] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index 4 to 5, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:31.126219 21439 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b"
dest_uuid: "e72358c248634f30be5f9becf0ec0fa0"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:51500
I20240502 02:04:31.126400 21439 raft_consensus.cc:595] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:31.135561 21708 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "2fd616a594f245a3b13353f1c9cb9064"
dest_uuid: "0983e154926c44ceb508483878dba5a0"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:38062
I20240502 02:04:31.135697 21708 raft_consensus.cc:595] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:31.339892 23532 raft_consensus.cc:979] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: : Instructing follower 2590825fe67b48c9adb5894c322237dc to start an election
I20240502 02:04:31.340010 23532 raft_consensus.cc:1067] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Signalling peer 2590825fe67b48c9adb5894c322237dc to start an election
I20240502 02:04:31.340571 22582 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "2fd616a594f245a3b13353f1c9cb9064"
dest_uuid: "2590825fe67b48c9adb5894c322237dc"
from {username='slave'} at 127.7.221.67:44097
I20240502 02:04:31.340773 22582 raft_consensus.cc:482] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 02:04:31.340889 22582 raft_consensus.cc:3037] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:31.341990 22582 raft_consensus.cc:504] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } }
I20240502 02:04:31.343222 22582 leader_election.cc:288] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [CANDIDATE]: Term 2 election: Requested vote from peers 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219), e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519), 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:31.343482 21439 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2fd616a594f245a3b13353f1c9cb9064" candidate_uuid: "2590825fe67b48c9adb5894c322237dc" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "e72358c248634f30be5f9becf0ec0fa0"
I20240502 02:04:31.343603 21439 raft_consensus.cc:3037] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:31.343582 21708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2fd616a594f245a3b13353f1c9cb9064" candidate_uuid: "2590825fe67b48c9adb5894c322237dc" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "0983e154926c44ceb508483878dba5a0"
I20240502 02:04:31.343686 21708 raft_consensus.cc:3032] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 02:04:31.343724 21708 raft_consensus.cc:729] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 0983e154926c44ceb508483878dba5a0, State: Running, Role: LEADER
I20240502 02:04:31.343724 21573 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2fd616a594f245a3b13353f1c9cb9064" candidate_uuid: "2590825fe67b48c9adb5894c322237dc" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "252575ab96ac4a079bab6ade63792782"
I20240502 02:04:31.343871 21573 raft_consensus.cc:3037] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:31.343879 21708 consensus_queue.cc:260] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } }
I20240502 02:04:31.344043 21708 raft_consensus.cc:3037] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:31.344610 21439 raft_consensus.cc:2454] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2590825fe67b48c9adb5894c322237dc in term 2.
I20240502 02:04:31.344988 21573 raft_consensus.cc:2454] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2590825fe67b48c9adb5894c322237dc in term 2.
I20240502 02:04:31.345072 21708 raft_consensus.cc:2454] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2590825fe67b48c9adb5894c322237dc in term 2.
I20240502 02:04:31.345376 22517 leader_election.cc:302] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 252575ab96ac4a079bab6ade63792782, 2590825fe67b48c9adb5894c322237dc, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:31.345695 23592 raft_consensus.cc:2781] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [term 2 FOLLOWER]: Leader election won for term 2
I20240502 02:04:31.345857 23592 raft_consensus.cc:686] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [term 2 LEADER]: Becoming Leader. State: Replica: 2590825fe67b48c9adb5894c322237dc, State: Running, Role: LEADER
I20240502 02:04:31.346190 23592 consensus_queue.cc:237] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } }
I20240502 02:04:31.348793 21290 catalog_manager.cc:5484] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc reported cstate change: term changed from 1 to 2, leader changed from 0983e154926c44ceb508483878dba5a0 (127.7.221.67) to 2590825fe67b48c9adb5894c322237dc (127.7.221.70). New cstate: current_term: 2 leader_uuid: "2590825fe67b48c9adb5894c322237dc" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:31.349200 21439 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b"
dest_uuid: "e72358c248634f30be5f9becf0ec0fa0"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:51528
I20240502 02:04:31.349306 21439 raft_consensus.cc:595] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:31.545630 23557 raft_consensus.cc:979] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0: : Instructing follower d590702730234f799e2a26de695bd7a2 to start an election
I20240502 02:04:31.545748 23557 raft_consensus.cc:1067] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Signalling peer d590702730234f799e2a26de695bd7a2 to start an election
I20240502 02:04:31.546258 22718 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b"
dest_uuid: "d590702730234f799e2a26de695bd7a2"
from {username='slave'} at 127.7.221.65:40421
I20240502 02:04:31.546437 22718 raft_consensus.cc:482] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 02:04:31.546556 22718 raft_consensus.cc:3037] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:31.547508 22718 raft_consensus.cc:504] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } }
I20240502 02:04:31.548558 22718 leader_election.cc:288] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [CANDIDATE]: Term 2 election: Requested vote from peers 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219), e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519), 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:31.549046 21708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b" candidate_uuid: "d590702730234f799e2a26de695bd7a2" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "0983e154926c44ceb508483878dba5a0"
I20240502 02:04:31.549094 21439 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b" candidate_uuid: "d590702730234f799e2a26de695bd7a2" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "e72358c248634f30be5f9becf0ec0fa0"
I20240502 02:04:31.549180 21708 raft_consensus.cc:3037] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:31.549196 21439 raft_consensus.cc:3032] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 02:04:31.549237 21439 raft_consensus.cc:729] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Running, Role: LEADER
I20240502 02:04:31.549408 21439 consensus_queue.cc:260] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } }
I20240502 02:04:31.549583 21439 raft_consensus.cc:3037] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:31.550159 21708 raft_consensus.cc:2454] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d590702730234f799e2a26de695bd7a2 in term 2.
I20240502 02:04:31.550514 21439 raft_consensus.cc:2454] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d590702730234f799e2a26de695bd7a2 in term 2.
I20240502 02:04:31.550812 22652 leader_election.cc:302] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 0983e154926c44ceb508483878dba5a0, d590702730234f799e2a26de695bd7a2, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:31.551136 23602 raft_consensus.cc:2781] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 02:04:31.551303 23602 raft_consensus.cc:686] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 2 LEADER]: Becoming Leader. State: Replica: d590702730234f799e2a26de695bd7a2, State: Running, Role: LEADER
I20240502 02:04:31.551594 23602 consensus_queue.cc:237] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } }
I20240502 02:04:31.552700 21573 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b" candidate_uuid: "d590702730234f799e2a26de695bd7a2" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "252575ab96ac4a079bab6ade63792782"
I20240502 02:04:31.552839 21573 raft_consensus.cc:3037] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:31.553881 21573 raft_consensus.cc:2454] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d590702730234f799e2a26de695bd7a2 in term 2.
I20240502 02:04:31.554499 21290 catalog_manager.cc:5484] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 reported cstate change: term changed from 1 to 2, leader changed from e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) to d590702730234f799e2a26de695bd7a2 (127.7.221.68). New cstate: current_term: 2 leader_uuid: "d590702730234f799e2a26de695bd7a2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:31.733071 21708 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Refusing update from remote peer 2590825fe67b48c9adb5894c322237dc: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 02:04:31.733584 23592 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:31.854987 21439 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Refusing update from remote peer 2590825fe67b48c9adb5894c322237dc: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 02:04:31.855448 23592 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:31.859611 21573 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Refusing update from remote peer 2590825fe67b48c9adb5894c322237dc: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 02:04:31.860569 23592 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:31.995739 21573 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Refusing update from remote peer d590702730234f799e2a26de695bd7a2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 02:04:31.996196 23602 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:32.032966 21439 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Refusing update from remote peer d590702730234f799e2a26de695bd7a2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 02:04:32.033479 23602 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:32.037034 21708 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Refusing update from remote peer d590702730234f799e2a26de695bd7a2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 02:04:32.038872 23603 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:32.470005 22582 consensus_queue.cc:237] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } }
I20240502 02:04:32.471220 21573 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Refusing update from remote peer 2590825fe67b48c9adb5894c322237dc: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240502 02:04:32.471302 21439 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Refusing update from remote peer 2590825fe67b48c9adb5894c322237dc: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240502 02:04:32.471684 23672 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 02:04:32.471917 23671 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 02:04:32.473487 23672 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:32.473912 21573 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:32.475047 21281 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2fd616a594f245a3b13353f1c9cb9064 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:32.475576 21439 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:32.476078 21290 catalog_manager.cc:5484] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc reported cstate change: config changed from index 5 to 7, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New cstate: current_term: 2 leader_uuid: "2590825fe67b48c9adb5894c322237dc" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:32.479276 21687 tablet_service.cc:1514] Processing DeleteTablet for tablet 2fd616a594f245a3b13353f1c9cb9064 with delete_type TABLET_DATA_TOMBSTONED (TS 0983e154926c44ceb508483878dba5a0 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:45734
I20240502 02:04:32.479604 23677 tablet_replica.cc:330] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: stopping tablet replica
I20240502 02:04:32.479750 23677 raft_consensus.cc:2229] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:32.479867 23677 raft_consensus.cc:2258] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:32.480437 23677 ts_tablet_manager.cc:1875] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:32.482156 23677 ts_tablet_manager.cc:1888] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 02:04:32.482223 23677 log.cc:1196] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/2fd616a594f245a3b13353f1c9cb9064
I20240502 02:04:32.482721 21283 catalog_manager.cc:4830] TS 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557): tablet 2fd616a594f245a3b13353f1c9cb9064 (table rebalance_test_table_0 [id=e1df1cf914834ab4bf5ac68bc944cbcc]) successfully deleted
I20240502 02:04:32.575336 22718 consensus_queue.cc:237] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } }
I20240502 02:04:32.576398 21708 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Refusing update from remote peer d590702730234f799e2a26de695bd7a2: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240502 02:04:32.576452 21573 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Refusing update from remote peer d590702730234f799e2a26de695bd7a2: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240502 02:04:32.576824 23602 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 02:04:32.577039 23678 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 02:04:32.578567 23679 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:32.579061 21708 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:32.579087 21573 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:32.580075 21282 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet bb5988c6c4c14f46a5e598cc1a59c99b with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:32.581313 21294 catalog_manager.cc:5484] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 reported cstate change: config changed from index 5 to 7, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New cstate: current_term: 2 leader_uuid: "d590702730234f799e2a26de695bd7a2" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:32.583746 21417 tablet_service.cc:1514] Processing DeleteTablet for tablet bb5988c6c4c14f46a5e598cc1a59c99b with delete_type TABLET_DATA_TOMBSTONED (TS e72358c248634f30be5f9becf0ec0fa0 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:56444
I20240502 02:04:32.584091 23680 tablet_replica.cc:330] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0: stopping tablet replica
I20240502 02:04:32.584239 23680 raft_consensus.cc:2229] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:32.584329 23680 raft_consensus.cc:2258] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:32.584918 23680 ts_tablet_manager.cc:1875] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:32.586562 23680 ts_tablet_manager.cc:1888] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 02:04:32.586690 23680 log.cc:1196] T bb5988c6c4c14f46a5e598cc1a59c99b P e72358c248634f30be5f9becf0ec0fa0: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/bb5988c6c4c14f46a5e598cc1a59c99b
I20240502 02:04:32.587208 21282 catalog_manager.cc:4830] TS e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519): tablet bb5988c6c4c14f46a5e598cc1a59c99b (table rebalance_test_table_2 [id=c5c742d762c44789b376f2c4c64c13b2]) successfully deleted
I20240502 02:04:32.678191 21687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 02:04:32.678851 22562 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:32.679056 21551 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:32.686795 21417 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 02:04:32.687952 22697 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:32.693257 22428 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:32.782019 22562 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:32.782716 21551 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:32.782836 22428 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:32.784152 21687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 02:04:32.785548 21417 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 02:04:32.799022 22697 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:32.831450 21439 consensus_queue.cc:237] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: true } }
I20240502 02:04:32.833024 21708 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 02:04:32.833079 22449 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 02:04:32.833642 21372 consensus_peers.cc:485] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 -> Peer 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219): Couldn't send request to peer 252575ab96ac4a079bab6ade63792782. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:32.834025 23780 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 02:04:32.834419 23780 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 02:04:32.836058 23779 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: true } } }
I20240502 02:04:32.837024 22449 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: true } } }
I20240502 02:04:32.838240 21708 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: true } } }
I20240502 02:04:32.839658 21294 catalog_manager.cc:5484] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: config changed from index 6 to 7, NON_VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) added. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:32.846382 21440 consensus_queue.cc:237] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: true } }
I20240502 02:04:32.847976 21708 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 02:04:32.847991 22449 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 02:04:32.848330 23557 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 02:04:32.848635 21372 consensus_peers.cc:485] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 -> Peer 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219): Couldn't send request to peer 252575ab96ac4a079bab6ade63792782. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:32.848690 23557 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 02:04:32.850487 23780 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: true } } }
I20240502 02:04:32.851346 22449 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: true } } }
I20240502 02:04:32.852355 21708 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: true } } }
I20240502 02:04:32.854393 21290 catalog_manager.cc:5484] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: config changed from index 6 to 7, NON_VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) added. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:32.866959 21708 consensus_queue.cc:237] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: true } }
I20240502 02:04:32.868851 22718 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 02:04:32.868938 21573 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 02:04:32.869345 23813 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 02:04:32.869603 21642 consensus_peers.cc:485] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 -> Peer e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519): Couldn't send request to peer e72358c248634f30be5f9becf0ec0fa0. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:32.869804 23649 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 02:04:32.871367 23813 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: true } } }
I20240502 02:04:32.871785 22718 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: true } } }
I20240502 02:04:32.872215 21573 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: true } } }
I20240502 02:04:32.874418 21292 catalog_manager.cc:5484] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 reported cstate change: config changed from index 6 to 7, NON_VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) added. New cstate: current_term: 1 leader_uuid: "0983e154926c44ceb508483878dba5a0" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:32.888257 22582 consensus_queue.cc:237] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 2.7, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: true } }
I20240502 02:04:32.889698 21440 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Refusing update from remote peer 2590825fe67b48c9adb5894c322237dc: Log matching property violated. Preceding OpId in replica: term: 2 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
I20240502 02:04:32.890148 21573 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Refusing update from remote peer 2590825fe67b48c9adb5894c322237dc: Log matching property violated. Preceding OpId in replica: term: 2 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
I20240502 02:04:32.890400 23672 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240502 02:04:32.890852 22518 consensus_peers.cc:485] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc -> Peer 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557): Couldn't send request to peer 0983e154926c44ceb508483878dba5a0. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:32.891597 23672 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:32.892426 23672 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [term 2 LEADER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: true } } }
I20240502 02:04:32.893013 21440 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: true } } }
I20240502 02:04:32.893682 21573 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: true } } }
I20240502 02:04:32.895165 21290 catalog_manager.cc:5484] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: config changed from index 7 to 8, NON_VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) added. New cstate: current_term: 2 leader_uuid: "2590825fe67b48c9adb5894c322237dc" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: true } } }
I20240502 02:04:32.907056 21551 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:32.907286 21687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 02:04:32.908372 22697 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:32.914125 22428 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:32.920517 22562 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:32.920792 21417 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 02:04:32.957881 21439 consensus_queue.cc:237] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } }
I20240502 02:04:32.959507 22449 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 02:04:32.959569 21373 consensus_peers.cc:485] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 -> Peer 2590825fe67b48c9adb5894c322237dc (127.7.221.70:39691): Couldn't send request to peer 2590825fe67b48c9adb5894c322237dc. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6cd9e4d44fdb40448877d5560a58c9f4. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:32.959726 21708 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 02:04:32.959887 23557 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 02:04:32.960075 23779 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 02:04:32.961884 23780 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } } }
I20240502 02:04:32.962343 22449 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } } }
I20240502 02:04:32.962500 21708 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } } }
I20240502 02:04:32.964709 21294 catalog_manager.cc:5484] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: config changed from index 6 to 7, NON_VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) added. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:32.971081 22718 consensus_queue.cc:237] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 2.7, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } }
I20240502 02:04:32.972285 21708 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Refusing update from remote peer d590702730234f799e2a26de695bd7a2: Log matching property violated. Preceding OpId in replica: term: 2 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
I20240502 02:04:32.972647 21573 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Refusing update from remote peer d590702730234f799e2a26de695bd7a2: Log matching property violated. Preceding OpId in replica: term: 2 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
I20240502 02:04:32.972642 23678 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:32.973024 23679 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:32.974406 23679 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 2 LEADER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:32.976258 21708 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
W20240502 02:04:32.976665 22650 consensus_peers.cc:485] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 -> Peer 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69:45619): Couldn't send request to peer 5e9df19de7264d1a9ee930ee770a8701. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bb5988c6c4c14f46a5e598cc1a59c99b. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:32.976565 21573 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:32.977245 21290 catalog_manager.cc:5484] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 reported cstate change: config changed from index 7 to 8, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New cstate: current_term: 2 leader_uuid: "d590702730234f799e2a26de695bd7a2" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:32.986930 21573 consensus_queue.cc:237] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } }
I20240502 02:04:32.988408 22582 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 02:04:32.988449 21439 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 02:04:32.988878 23881 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 02:04:32.989120 23880 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 02:04:32.990854 23880 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:32.991294 21439 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:32.993160 21294 catalog_manager.cc:5484] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index 6 to 7, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New cstate: current_term: 1 leader_uuid: "252575ab96ac4a079bab6ade63792782" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 02:04:32.993582 21505 consensus_peers.cc:485] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 -> Peer 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69:45619): Couldn't send request to peer 5e9df19de7264d1a9ee930ee770a8701. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1683e87385ab4050824a71508d3c3f5b. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:32.994437 22582 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } } }
I20240502 02:04:33.340492 23937 ts_tablet_manager.cc:902] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: Initiating tablet copy from peer 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557)
I20240502 02:04:33.341207 23937 tablet_copy_client.cc:308] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:35557
I20240502 02:04:33.341770 21728 tablet_copy_service.cc:140] P 0983e154926c44ceb508483878dba5a0: Received BeginTabletCopySession request for tablet fd1b849d9ecd4f5e96b678673444f4d6 from peer e72358c248634f30be5f9becf0ec0fa0 ({username='slave'} at 127.7.221.65:43211)
I20240502 02:04:33.341889 21728 tablet_copy_service.cc:161] P 0983e154926c44ceb508483878dba5a0: Beginning new tablet copy session on tablet fd1b849d9ecd4f5e96b678673444f4d6 from peer e72358c248634f30be5f9becf0ec0fa0 at {username='slave'} at 127.7.221.65:43211: session id = e72358c248634f30be5f9becf0ec0fa0-fd1b849d9ecd4f5e96b678673444f4d6
I20240502 02:04:33.342654 21728 tablet_copy_source_session.cc:205] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:33.343642 23937 ts_tablet_manager.cc:1875] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 02:04:33.345378 23937 ts_tablet_manager.cc:1888] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.5
I20240502 02:04:33.345561 23937 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fd1b849d9ecd4f5e96b678673444f4d6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:33.346462 23937 tablet_copy_client.cc:783] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:33.346688 23937 tablet_copy_client.cc:647] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:33.348084 23937 tablet_copy_client.cc:523] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:33.349306 23937 tablet_bootstrap.cc:492] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: Bootstrap starting.
I20240502 02:04:33.367414 23937 tablet_bootstrap.cc:492] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:33.367702 23937 tablet_bootstrap.cc:492] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: Bootstrap complete.
I20240502 02:04:33.367882 23937 ts_tablet_manager.cc:1367] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: Time spent bootstrapping tablet: real 0.019s user 0.018s sys 0.001s
I20240502 02:04:33.368271 23937 raft_consensus.cc:348] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: true } }
I20240502 02:04:33.368372 23937 raft_consensus.cc:729] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Initialized, Role: LEARNER
I20240502 02:04:33.368464 23937 consensus_queue.cc:260] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: NON_VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: true } }
I20240502 02:04:33.368719 23937 ts_tablet_manager.cc:1398] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:33.369307 21728 tablet_copy_service.cc:342] P 0983e154926c44ceb508483878dba5a0: Request end of tablet copy session e72358c248634f30be5f9becf0ec0fa0-fd1b849d9ecd4f5e96b678673444f4d6 received from {username='slave'} at 127.7.221.65:43211
I20240502 02:04:33.369390 21728 tablet_copy_service.cc:433] P 0983e154926c44ceb508483878dba5a0: ending tablet copy session e72358c248634f30be5f9becf0ec0fa0-fd1b849d9ecd4f5e96b678673444f4d6 on tablet fd1b849d9ecd4f5e96b678673444f4d6 with peer e72358c248634f30be5f9becf0ec0fa0
I20240502 02:04:33.383433 23948 ts_tablet_manager.cc:902] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: Initiating tablet copy from peer e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519)
I20240502 02:04:33.384259 23948 tablet_copy_client.cc:308] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:38519
I20240502 02:04:33.384913 21460 tablet_copy_service.cc:140] P e72358c248634f30be5f9becf0ec0fa0: Received BeginTabletCopySession request for tablet c0db5fac24d54971baf964b55e62abe4 from peer 252575ab96ac4a079bab6ade63792782 ({username='slave'} at 127.7.221.66:37987)
I20240502 02:04:33.385015 21460 tablet_copy_service.cc:161] P e72358c248634f30be5f9becf0ec0fa0: Beginning new tablet copy session on tablet c0db5fac24d54971baf964b55e62abe4 from peer 252575ab96ac4a079bab6ade63792782 at {username='slave'} at 127.7.221.66:37987: session id = 252575ab96ac4a079bab6ade63792782-c0db5fac24d54971baf964b55e62abe4
I20240502 02:04:33.385831 21460 tablet_copy_source_session.cc:205] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:33.386842 23948 ts_tablet_manager.cc:1875] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 02:04:33.388844 23948 ts_tablet_manager.cc:1888] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.5
I20240502 02:04:33.389017 23948 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c0db5fac24d54971baf964b55e62abe4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:33.389973 23948 tablet_copy_client.cc:783] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:33.390147 23948 tablet_copy_client.cc:647] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:33.391671 23948 tablet_copy_client.cc:523] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:33.393112 23948 tablet_bootstrap.cc:492] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: Bootstrap starting.
I20240502 02:04:33.413285 23948 tablet_bootstrap.cc:492] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:33.413594 23948 tablet_bootstrap.cc:492] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: Bootstrap complete.
I20240502 02:04:33.413758 23948 ts_tablet_manager.cc:1367] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: Time spent bootstrapping tablet: real 0.021s user 0.019s sys 0.000s
I20240502 02:04:33.414160 23948 raft_consensus.cc:348] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: true } }
I20240502 02:04:33.414278 23948 raft_consensus.cc:729] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 252575ab96ac4a079bab6ade63792782, State: Initialized, Role: LEARNER
I20240502 02:04:33.414386 23948 consensus_queue.cc:260] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: true } }
I20240502 02:04:33.414702 23948 ts_tablet_manager.cc:1398] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782: Time spent starting tablet: real 0.001s user 0.004s sys 0.000s
I20240502 02:04:33.415707 21460 tablet_copy_service.cc:342] P e72358c248634f30be5f9becf0ec0fa0: Request end of tablet copy session 252575ab96ac4a079bab6ade63792782-c0db5fac24d54971baf964b55e62abe4 received from {username='slave'} at 127.7.221.66:37987
I20240502 02:04:33.415800 21460 tablet_copy_service.cc:433] P e72358c248634f30be5f9becf0ec0fa0: ending tablet copy session 252575ab96ac4a079bab6ade63792782-c0db5fac24d54971baf964b55e62abe4 on tablet c0db5fac24d54971baf964b55e62abe4 with peer 252575ab96ac4a079bab6ade63792782
I20240502 02:04:33.417721 23969 ts_tablet_manager.cc:902] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701: Initiating tablet copy from peer d590702730234f799e2a26de695bd7a2 (127.7.221.68:46455)
I20240502 02:04:33.418175 23969 tablet_copy_client.cc:308] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.68:46455
I20240502 02:04:33.421579 22738 tablet_copy_service.cc:140] P d590702730234f799e2a26de695bd7a2: Received BeginTabletCopySession request for tablet bb5988c6c4c14f46a5e598cc1a59c99b from peer 5e9df19de7264d1a9ee930ee770a8701 ({username='slave'} at 127.7.221.69:46077)
I20240502 02:04:33.421710 22738 tablet_copy_service.cc:161] P d590702730234f799e2a26de695bd7a2: Beginning new tablet copy session on tablet bb5988c6c4c14f46a5e598cc1a59c99b from peer 5e9df19de7264d1a9ee930ee770a8701 at {username='slave'} at 127.7.221.69:46077: session id = 5e9df19de7264d1a9ee930ee770a8701-bb5988c6c4c14f46a5e598cc1a59c99b
I20240502 02:04:33.422809 22738 tablet_copy_source_session.cc:205] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:33.423774 23969 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bb5988c6c4c14f46a5e598cc1a59c99b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:33.425498 23969 tablet_copy_client.cc:783] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:33.425642 23969 tablet_copy_client.cc:647] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:33.427101 23969 tablet_copy_client.cc:523] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:33.428360 23969 tablet_bootstrap.cc:492] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701: Bootstrap starting.
I20240502 02:04:33.428818 23981 ts_tablet_manager.cc:902] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701: Initiating tablet copy from peer 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219)
I20240502 02:04:33.429343 23981 tablet_copy_client.cc:308] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:34219
I20240502 02:04:33.438933 21595 tablet_copy_service.cc:140] P 252575ab96ac4a079bab6ade63792782: Received BeginTabletCopySession request for tablet 1683e87385ab4050824a71508d3c3f5b from peer 5e9df19de7264d1a9ee930ee770a8701 ({username='slave'} at 127.7.221.69:36809)
I20240502 02:04:33.439062 21595 tablet_copy_service.cc:161] P 252575ab96ac4a079bab6ade63792782: Beginning new tablet copy session on tablet 1683e87385ab4050824a71508d3c3f5b from peer 5e9df19de7264d1a9ee930ee770a8701 at {username='slave'} at 127.7.221.69:36809: session id = 5e9df19de7264d1a9ee930ee770a8701-1683e87385ab4050824a71508d3c3f5b
I20240502 02:04:33.439957 21595 tablet_copy_source_session.cc:205] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:33.440941 23981 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1683e87385ab4050824a71508d3c3f5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:33.443192 23981 tablet_copy_client.cc:783] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:33.443341 23981 tablet_copy_client.cc:647] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:33.444373 23981 tablet_copy_client.cc:523] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:33.445618 23981 tablet_bootstrap.cc:492] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701: Bootstrap starting.
I20240502 02:04:33.451592 23969 tablet_bootstrap.cc:492] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:33.451990 23969 tablet_bootstrap.cc:492] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701: Bootstrap complete.
I20240502 02:04:33.452191 23969 ts_tablet_manager.cc:1367] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701: Time spent bootstrapping tablet: real 0.024s user 0.024s sys 0.000s
I20240502 02:04:33.452605 23969 raft_consensus.cc:348] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701 [term 2 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } }
I20240502 02:04:33.452723 23969 raft_consensus.cc:729] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701 [term 2 LEARNER]: Becoming Follower/Learner. State: Replica: 5e9df19de7264d1a9ee930ee770a8701, State: Initialized, Role: LEARNER
I20240502 02:04:33.452893 23969 consensus_queue.cc:260] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } }
I20240502 02:04:33.453157 23969 ts_tablet_manager.cc:1398] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:33.453685 22738 tablet_copy_service.cc:342] P d590702730234f799e2a26de695bd7a2: Request end of tablet copy session 5e9df19de7264d1a9ee930ee770a8701-bb5988c6c4c14f46a5e598cc1a59c99b received from {username='slave'} at 127.7.221.69:46077
I20240502 02:04:33.453789 22738 tablet_copy_service.cc:433] P d590702730234f799e2a26de695bd7a2: ending tablet copy session 5e9df19de7264d1a9ee930ee770a8701-bb5988c6c4c14f46a5e598cc1a59c99b on tablet bb5988c6c4c14f46a5e598cc1a59c99b with peer 5e9df19de7264d1a9ee930ee770a8701
I20240502 02:04:33.465021 23948 ts_tablet_manager.cc:902] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: Initiating tablet copy from peer e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519)
I20240502 02:04:33.465523 23948 tablet_copy_client.cc:308] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:38519
I20240502 02:04:33.465961 21460 tablet_copy_service.cc:140] P e72358c248634f30be5f9becf0ec0fa0: Received BeginTabletCopySession request for tablet a22071eab1054f0ebc97dfd4e0515148 from peer 252575ab96ac4a079bab6ade63792782 ({username='slave'} at 127.7.221.66:37987)
I20240502 02:04:33.466066 21460 tablet_copy_service.cc:161] P e72358c248634f30be5f9becf0ec0fa0: Beginning new tablet copy session on tablet a22071eab1054f0ebc97dfd4e0515148 from peer 252575ab96ac4a079bab6ade63792782 at {username='slave'} at 127.7.221.66:37987: session id = 252575ab96ac4a079bab6ade63792782-a22071eab1054f0ebc97dfd4e0515148
I20240502 02:04:33.466982 21460 tablet_copy_source_session.cc:205] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:33.467235 24004 ts_tablet_manager.cc:902] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: Initiating tablet copy from peer 2590825fe67b48c9adb5894c322237dc (127.7.221.70:39691)
I20240502 02:04:33.467751 23948 ts_tablet_manager.cc:1875] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 02:04:33.468029 24004 tablet_copy_client.cc:308] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.70:39691
I20240502 02:04:33.468734 22603 tablet_copy_service.cc:140] P 2590825fe67b48c9adb5894c322237dc: Received BeginTabletCopySession request for tablet 2fd616a594f245a3b13353f1c9cb9064 from peer 0983e154926c44ceb508483878dba5a0 ({username='slave'} at 127.7.221.67:44097)
I20240502 02:04:33.468863 22603 tablet_copy_service.cc:161] P 2590825fe67b48c9adb5894c322237dc: Beginning new tablet copy session on tablet 2fd616a594f245a3b13353f1c9cb9064 from peer 0983e154926c44ceb508483878dba5a0 at {username='slave'} at 127.7.221.67:44097: session id = 0983e154926c44ceb508483878dba5a0-2fd616a594f245a3b13353f1c9cb9064
I20240502 02:04:33.469530 23981 tablet_bootstrap.cc:492] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:33.469638 23948 ts_tablet_manager.cc:1888] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.5
I20240502 02:04:33.469758 23948 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a22071eab1054f0ebc97dfd4e0515148. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:33.469801 23981 tablet_bootstrap.cc:492] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701: Bootstrap complete.
I20240502 02:04:33.469956 23981 ts_tablet_manager.cc:1367] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701: Time spent bootstrapping tablet: real 0.024s user 0.019s sys 0.004s
I20240502 02:04:33.470255 22603 tablet_copy_source_session.cc:205] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:33.470415 23981 raft_consensus.cc:348] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } }
I20240502 02:04:33.470504 23948 tablet_copy_client.cc:783] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:33.470517 23981 raft_consensus.cc:729] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 5e9df19de7264d1a9ee930ee770a8701, State: Initialized, Role: LEARNER
I20240502 02:04:33.470594 23948 tablet_copy_client.cc:647] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:33.470664 23981 consensus_queue.cc:260] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: NON_VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: true } }
I20240502 02:04:33.471091 23981 ts_tablet_manager.cc:1398] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701: Time spent starting tablet: real 0.001s user 0.003s sys 0.000s
I20240502 02:04:33.471619 24004 ts_tablet_manager.cc:1875] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 02:04:33.471794 23948 tablet_copy_client.cc:523] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:33.472353 21595 tablet_copy_service.cc:342] P 252575ab96ac4a079bab6ade63792782: Request end of tablet copy session 5e9df19de7264d1a9ee930ee770a8701-1683e87385ab4050824a71508d3c3f5b received from {username='slave'} at 127.7.221.69:36809
I20240502 02:04:33.472421 21595 tablet_copy_service.cc:433] P 252575ab96ac4a079bab6ade63792782: ending tablet copy session 5e9df19de7264d1a9ee930ee770a8701-1683e87385ab4050824a71508d3c3f5b on tablet 1683e87385ab4050824a71508d3c3f5b with peer 5e9df19de7264d1a9ee930ee770a8701
I20240502 02:04:33.472729 23948 tablet_bootstrap.cc:492] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: Bootstrap starting.
I20240502 02:04:33.474063 24004 ts_tablet_manager.cc:1888] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 2.6
I20240502 02:04:33.474200 24004 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2fd616a594f245a3b13353f1c9cb9064. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:33.475080 24004 tablet_copy_client.cc:783] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:33.475256 24004 tablet_copy_client.cc:647] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:33.476763 24004 tablet_copy_client.cc:523] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:33.478022 24004 tablet_bootstrap.cc:492] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: Bootstrap starting.
I20240502 02:04:33.489262 23948 tablet_bootstrap.cc:492] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:33.489483 23948 tablet_bootstrap.cc:492] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: Bootstrap complete.
I20240502 02:04:33.489622 23948 ts_tablet_manager.cc:1367] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: Time spent bootstrapping tablet: real 0.017s user 0.013s sys 0.002s
I20240502 02:04:33.489985 23948 raft_consensus.cc:348] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: true } }
I20240502 02:04:33.490085 23948 raft_consensus.cc:729] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 252575ab96ac4a079bab6ade63792782, State: Initialized, Role: LEARNER
I20240502 02:04:33.490222 23948 consensus_queue.cc:260] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: NON_VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: true } }
I20240502 02:04:33.490459 23948 ts_tablet_manager.cc:1398] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782: Time spent starting tablet: real 0.001s user 0.003s sys 0.000s
I20240502 02:04:33.491079 21460 tablet_copy_service.cc:342] P e72358c248634f30be5f9becf0ec0fa0: Request end of tablet copy session 252575ab96ac4a079bab6ade63792782-a22071eab1054f0ebc97dfd4e0515148 received from {username='slave'} at 127.7.221.66:37987
I20240502 02:04:33.491182 21460 tablet_copy_service.cc:433] P e72358c248634f30be5f9becf0ec0fa0: ending tablet copy session 252575ab96ac4a079bab6ade63792782-a22071eab1054f0ebc97dfd4e0515148 on tablet a22071eab1054f0ebc97dfd4e0515148 with peer 252575ab96ac4a079bab6ade63792782
I20240502 02:04:33.496665 24004 tablet_bootstrap.cc:492] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:33.496908 24004 tablet_bootstrap.cc:492] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: Bootstrap complete.
I20240502 02:04:33.497037 24004 ts_tablet_manager.cc:1367] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: Time spent bootstrapping tablet: real 0.019s user 0.015s sys 0.002s
I20240502 02:04:33.497349 24004 raft_consensus.cc:348] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 2 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: true } }
I20240502 02:04:33.497445 24004 raft_consensus.cc:729] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 2 LEARNER]: Becoming Follower/Learner. State: Replica: 0983e154926c44ceb508483878dba5a0, State: Initialized, Role: LEARNER
I20240502 02:04:33.497521 24004 consensus_queue.cc:260] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: NON_VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: true } }
I20240502 02:04:33.497761 24004 ts_tablet_manager.cc:1398] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:33.498323 22603 tablet_copy_service.cc:342] P 2590825fe67b48c9adb5894c322237dc: Request end of tablet copy session 0983e154926c44ceb508483878dba5a0-2fd616a594f245a3b13353f1c9cb9064 received from {username='slave'} at 127.7.221.67:44097
I20240502 02:04:33.498405 22603 tablet_copy_service.cc:433] P 2590825fe67b48c9adb5894c322237dc: ending tablet copy session 0983e154926c44ceb508483878dba5a0-2fd616a594f245a3b13353f1c9cb9064 on tablet 2fd616a594f245a3b13353f1c9cb9064 with peer 0983e154926c44ceb508483878dba5a0
I20240502 02:04:33.513603 24011 ts_tablet_manager.cc:902] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc: Initiating tablet copy from peer e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519)
I20240502 02:04:33.514037 24011 tablet_copy_client.cc:308] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:38519
I20240502 02:04:33.514461 21460 tablet_copy_service.cc:140] P e72358c248634f30be5f9becf0ec0fa0: Received BeginTabletCopySession request for tablet 6cd9e4d44fdb40448877d5560a58c9f4 from peer 2590825fe67b48c9adb5894c322237dc ({username='slave'} at 127.7.221.70:35727)
I20240502 02:04:33.514559 21460 tablet_copy_service.cc:161] P e72358c248634f30be5f9becf0ec0fa0: Beginning new tablet copy session on tablet 6cd9e4d44fdb40448877d5560a58c9f4 from peer 2590825fe67b48c9adb5894c322237dc at {username='slave'} at 127.7.221.70:35727: session id = 2590825fe67b48c9adb5894c322237dc-6cd9e4d44fdb40448877d5560a58c9f4
I20240502 02:04:33.515311 21460 tablet_copy_source_session.cc:205] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:33.516090 24011 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6cd9e4d44fdb40448877d5560a58c9f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:33.517769 24011 tablet_copy_client.cc:783] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:33.517947 24011 tablet_copy_client.cc:647] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:33.518982 24011 tablet_copy_client.cc:523] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:33.519946 24011 tablet_bootstrap.cc:492] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc: Bootstrap starting.
I20240502 02:04:33.543002 24011 tablet_bootstrap.cc:492] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:33.543344 24011 tablet_bootstrap.cc:492] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc: Bootstrap complete.
I20240502 02:04:33.543519 24011 ts_tablet_manager.cc:1367] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc: Time spent bootstrapping tablet: real 0.024s user 0.023s sys 0.000s
I20240502 02:04:33.543938 24011 raft_consensus.cc:348] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } }
I20240502 02:04:33.544064 24011 raft_consensus.cc:729] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 2590825fe67b48c9adb5894c322237dc, State: Initialized, Role: LEARNER
I20240502 02:04:33.544159 24011 consensus_queue.cc:260] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: NON_VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: true } }
I20240502 02:04:33.544456 24011 ts_tablet_manager.cc:1398] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:33.545047 21460 tablet_copy_service.cc:342] P e72358c248634f30be5f9becf0ec0fa0: Request end of tablet copy session 2590825fe67b48c9adb5894c322237dc-6cd9e4d44fdb40448877d5560a58c9f4 received from {username='slave'} at 127.7.221.70:35727
I20240502 02:04:33.545142 21460 tablet_copy_service.cc:433] P e72358c248634f30be5f9becf0ec0fa0: ending tablet copy session 2590825fe67b48c9adb5894c322237dc-6cd9e4d44fdb40448877d5560a58c9f4 on tablet 6cd9e4d44fdb40448877d5560a58c9f4 with peer 2590825fe67b48c9adb5894c322237dc
I20240502 02:04:33.734100 21439 raft_consensus.cc:1203] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 02:04:33.866652 21708 raft_consensus.cc:1203] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 2 LEARNER]: Deduplicated request from leader. Original: 2.7->[2.8-2.8] Dedup: 2.8->[]
I20240502 02:04:33.927778 21573 raft_consensus.cc:1203] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 02:04:33.932461 22449 raft_consensus.cc:1203] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 02:04:33.934360 21573 raft_consensus.cc:1203] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 02:04:33.984969 22449 raft_consensus.cc:1203] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701 [term 2 LEARNER]: Deduplicated request from leader. Original: 2.7->[2.8-2.8] Dedup: 2.8->[]
I20240502 02:04:34.068663 22582 raft_consensus.cc:1203] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 02:04:34.118665 24052 raft_consensus.cc:1050] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0: attempting to promote NON_VOTER e72358c248634f30be5f9becf0ec0fa0 to VOTER
I20240502 02:04:34.119045 24052 consensus_queue.cc:237] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: false } }
I20240502 02:04:34.120311 21439 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEARNER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:34.120393 21573 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:34.120420 22718 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:34.120782 24135 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:34.121011 24053 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:34.121192 24052 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:34.123185 24053 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: false } } }
I20240502 02:04:34.123586 21439 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: false } } }
I20240502 02:04:34.123615 21573 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: false } } }
I20240502 02:04:34.125213 22718 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: false } } }
I20240502 02:04:34.126405 21295 catalog_manager.cc:5484] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 reported cstate change: config changed from index 7 to 8, e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "0983e154926c44ceb508483878dba5a0" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:34.129827 21708 consensus_queue.cc:237] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: false } }
I20240502 02:04:34.130939 21439 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 02:04:34.130998 22718 raft_consensus.cc:1261] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Refusing update from remote peer 0983e154926c44ceb508483878dba5a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 02:04:34.131330 23814 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:34.131517 24052 consensus_queue.cc:1035] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:34.132973 23814 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P 0983e154926c44ceb508483878dba5a0 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: false } } }
I20240502 02:04:34.133342 22718 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: false } } }
I20240502 02:04:34.133977 21439 raft_consensus.cc:2932] T fd1b849d9ecd4f5e96b678673444f4d6 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: false } } }
I20240502 02:04:34.135033 21283 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet fd1b849d9ecd4f5e96b678673444f4d6 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:34.136006 21290 catalog_manager.cc:5484] T fd1b849d9ecd4f5e96b678673444f4d6 P d590702730234f799e2a26de695bd7a2 reported cstate change: config changed from index 8 to 9, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "0983e154926c44ceb508483878dba5a0" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { promote: false } } }
I20240502 02:04:34.138080 21551 tablet_service.cc:1514] Processing DeleteTablet for tablet fd1b849d9ecd4f5e96b678673444f4d6 with delete_type TABLET_DATA_TOMBSTONED (TS 252575ab96ac4a079bab6ade63792782 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:43552
I20240502 02:04:34.138419 24143 tablet_replica.cc:330] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782: stopping tablet replica
I20240502 02:04:34.138566 24143 raft_consensus.cc:2229] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:34.138793 24143 raft_consensus.cc:2258] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:34.139492 24143 ts_tablet_manager.cc:1875] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:34.141031 24143 ts_tablet_manager.cc:1888] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 02:04:34.141112 24143 log.cc:1196] T fd1b849d9ecd4f5e96b678673444f4d6 P 252575ab96ac4a079bab6ade63792782: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/fd1b849d9ecd4f5e96b678673444f4d6
I20240502 02:04:34.141539 21280 catalog_manager.cc:4830] TS 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219): tablet fd1b849d9ecd4f5e96b678673444f4d6 (table rebalance_test_table_2 [id=c5c742d762c44789b376f2c4c64c13b2]) successfully deleted
I20240502 02:04:34.359697 24091 raft_consensus.cc:1050] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0: attempting to promote NON_VOTER 252575ab96ac4a079bab6ade63792782 to VOTER
I20240502 02:04:34.360141 24091 consensus_queue.cc:237] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } }
I20240502 02:04:34.361575 21573 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 LEARNER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:34.361707 22447 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:34.362062 21708 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:34.362079 24091 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:34.362232 24092 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:34.362506 24198 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:34.364627 24198 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 252575ab96ac4a079bab6ade63792782 (127.7.221.66) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:34.365181 21708 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 252575ab96ac4a079bab6ade63792782 (127.7.221.66) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:34.365334 21573 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 252575ab96ac4a079bab6ade63792782 (127.7.221.66) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:34.367151 22447 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 252575ab96ac4a079bab6ade63792782 (127.7.221.66) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:34.368376 21292 catalog_manager.cc:5484] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index 7 to 8, 252575ab96ac4a079bab6ade63792782 (127.7.221.66) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:34.370860 24202 raft_consensus.cc:1050] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc: attempting to promote NON_VOTER 0983e154926c44ceb508483878dba5a0 to VOTER
I20240502 02:04:34.371341 24202 consensus_queue.cc:237] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: false } }
I20240502 02:04:34.373061 21440 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Refusing update from remote peer 2590825fe67b48c9adb5894c322237dc: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 02:04:34.373137 21439 consensus_queue.cc:237] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } }
I20240502 02:04:34.373188 21573 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Refusing update from remote peer 2590825fe67b48c9adb5894c322237dc: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 02:04:34.373562 24202 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:34.374161 21573 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 02:04:34.374271 22447 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 02:04:34.374686 24202 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:34.375165 24091 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:34.375321 24092 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:34.375773 21708 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 2 LEARNER]: Refusing update from remote peer 2590825fe67b48c9adb5894c322237dc: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 02:04:34.376083 24075 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:34.376917 24092 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:34.377112 24202 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [term 2 LEADER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 0983e154926c44ceb508483878dba5a0 (127.7.221.67) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: false } } }
I20240502 02:04:34.377326 21573 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:34.378556 21282 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a22071eab1054f0ebc97dfd4e0515148 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:34.379371 21290 catalog_manager.cc:5484] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index 8 to 9, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:34.380451 21573 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 0983e154926c44ceb508483878dba5a0 (127.7.221.67) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: false } } }
I20240502 02:04:34.380736 21439 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 0983e154926c44ceb508483878dba5a0 (127.7.221.67) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: false } } }
I20240502 02:04:34.380808 21294 catalog_manager.cc:5484] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc reported cstate change: config changed from index 8 to 9, 0983e154926c44ceb508483878dba5a0 (127.7.221.67) changed from NON_VOTER to VOTER. New cstate: current_term: 2 leader_uuid: "2590825fe67b48c9adb5894c322237dc" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:34.382063 22447 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 0983e154926c44ceb508483878dba5a0 (127.7.221.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:34.384531 21687 tablet_service.cc:1514] Processing DeleteTablet for tablet a22071eab1054f0ebc97dfd4e0515148 with delete_type TABLET_DATA_TOMBSTONED (TS 0983e154926c44ceb508483878dba5a0 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:45734
I20240502 02:04:34.385151 21706 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 0983e154926c44ceb508483878dba5a0 (127.7.221.67) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { replace: true } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: false } } }
I20240502 02:04:34.386668 22582 consensus_queue.cc:237] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: false } }
I20240502 02:04:34.388003 24208 tablet_replica.cc:330] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0: stopping tablet replica
I20240502 02:04:34.388027 21439 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Refusing update from remote peer 2590825fe67b48c9adb5894c322237dc: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 02:04:34.388128 24208 raft_consensus.cc:2229] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:34.388355 21706 raft_consensus.cc:1261] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Refusing update from remote peer 2590825fe67b48c9adb5894c322237dc: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 02:04:34.388381 24202 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 02:04:34.388504 24208 raft_consensus.cc:2258] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:34.388793 24202 consensus_queue.cc:1035] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 02:04:34.389286 24208 ts_tablet_manager.cc:1875] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:34.390049 24075 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 9 to 10, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: false } } }
I20240502 02:04:34.390513 21439 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 9 to 10, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: false } } }
I20240502 02:04:34.391455 21281 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2fd616a594f245a3b13353f1c9cb9064 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:34.391744 24208 ts_tablet_manager.cc:1888] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 02:04:34.391809 24208 log.cc:1196] T a22071eab1054f0ebc97dfd4e0515148 P 0983e154926c44ceb508483878dba5a0: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/a22071eab1054f0ebc97dfd4e0515148
I20240502 02:04:34.392458 21294 catalog_manager.cc:5484] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc reported cstate change: config changed from index 9 to 10, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New cstate: current_term: 2 leader_uuid: "2590825fe67b48c9adb5894c322237dc" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:34.392753 21283 catalog_manager.cc:4830] TS 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557): tablet a22071eab1054f0ebc97dfd4e0515148 (table rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]) successfully deleted
I20240502 02:04:34.393036 21706 raft_consensus.cc:2932] T 2fd616a594f245a3b13353f1c9cb9064 P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 9 to 10, VOTER 252575ab96ac4a079bab6ade63792782 (127.7.221.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } attrs { promote: false } } }
I20240502 02:04:34.394843 21551 tablet_service.cc:1514] Processing DeleteTablet for tablet 2fd616a594f245a3b13353f1c9cb9064 with delete_type TABLET_DATA_TOMBSTONED (TS 252575ab96ac4a079bab6ade63792782 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:43552
I20240502 02:04:34.395087 24143 tablet_replica.cc:330] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782: stopping tablet replica
I20240502 02:04:34.395215 24143 raft_consensus.cc:2229] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:34.395407 24143 raft_consensus.cc:2258] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:34.396075 24143 ts_tablet_manager.cc:1875] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:34.397811 24143 ts_tablet_manager.cc:1888] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 02:04:34.397874 24143 log.cc:1196] T 2fd616a594f245a3b13353f1c9cb9064 P 252575ab96ac4a079bab6ade63792782: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/2fd616a594f245a3b13353f1c9cb9064
I20240502 02:04:34.398247 21280 catalog_manager.cc:4830] TS 252575ab96ac4a079bab6ade63792782 (127.7.221.66:34219): tablet 2fd616a594f245a3b13353f1c9cb9064 (table rebalance_test_table_0 [id=e1df1cf914834ab4bf5ac68bc944cbcc]) successfully deleted
I20240502 02:04:34.431555 24096 raft_consensus.cc:1050] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: attempting to promote NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 to VOTER
I20240502 02:04:34.431983 24096 consensus_queue.cc:237] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } }
I20240502 02:04:34.433454 22447 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701 [term 2 LEARNER]: Refusing update from remote peer d590702730234f799e2a26de695bd7a2: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 02:04:34.433558 21573 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Refusing update from remote peer d590702730234f799e2a26de695bd7a2: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 02:04:34.433558 21706 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Refusing update from remote peer d590702730234f799e2a26de695bd7a2: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 02:04:34.434036 24210 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:34.434492 24096 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:34.434707 23678 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:34.436897 24096 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 2 LEADER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:34.437317 22447 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:34.438192 21706 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:34.439188 21573 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:34.439738 21290 catalog_manager.cc:5484] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 reported cstate change: config changed from index 8 to 9, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 2 leader_uuid: "d590702730234f799e2a26de695bd7a2" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:34.517298 24091 raft_consensus.cc:1050] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0: attempting to promote NON_VOTER 252575ab96ac4a079bab6ade63792782 to VOTER
I20240502 02:04:34.517768 24091 consensus_queue.cc:237] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } }
I20240502 02:04:34.519389 21573 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 1 LEARNER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:34.519492 21706 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:34.519766 24197 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:34.519927 24091 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:34.522874 22447 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:34.523473 23779 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:34.525087 24092 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 252575ab96ac4a079bab6ade63792782 (127.7.221.66) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:34.527216 22447 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 252575ab96ac4a079bab6ade63792782 (127.7.221.66) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:34.529429 21292 catalog_manager.cc:5484] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 reported cstate change: config changed from index 7 to 8, 252575ab96ac4a079bab6ade63792782 (127.7.221.66) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:34.530113 21706 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 252575ab96ac4a079bab6ade63792782 (127.7.221.66) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:34.531088 24094 raft_consensus.cc:1050] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782: attempting to promote NON_VOTER 5e9df19de7264d1a9ee930ee770a8701 to VOTER
I20240502 02:04:34.531380 21573 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 252575ab96ac4a079bab6ade63792782 (127.7.221.66) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:34.531486 24094 consensus_queue.cc:237] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } }
I20240502 02:04:34.533175 22582 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:34.533202 21439 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:34.533607 24245 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:34.534457 24243 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:34.534502 22447 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701 [term 1 LEARNER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:34.535280 24094 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:34.538663 24094 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:34.539431 22582 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:34.540732 21295 catalog_manager.cc:5484] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index 7 to 8, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "252575ab96ac4a079bab6ade63792782" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:34.541635 21439 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:34.544361 22447 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:34.548111 21573 consensus_queue.cc:237] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } }
I20240502 02:04:34.549026 22447 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 02:04:34.549160 21439 raft_consensus.cc:1261] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 02:04:34.549540 24243 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:34.549723 24256 consensus_queue.cc:1035] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:34.551218 24243 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P 252575ab96ac4a079bab6ade63792782 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:34.551575 22447 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:34.552425 21280 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1683e87385ab4050824a71508d3c3f5b with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:34.552946 21439 raft_consensus.cc:2932] T 1683e87385ab4050824a71508d3c3f5b P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:34.553658 21294 catalog_manager.cc:5484] T 1683e87385ab4050824a71508d3c3f5b P 5e9df19de7264d1a9ee930ee770a8701 reported cstate change: config changed from index 8 to 9, VOTER 2590825fe67b48c9adb5894c322237dc (127.7.221.70) evicted. New cstate: current_term: 1 leader_uuid: "252575ab96ac4a079bab6ade63792782" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:34.556116 22562 tablet_service.cc:1514] Processing DeleteTablet for tablet 1683e87385ab4050824a71508d3c3f5b with delete_type TABLET_DATA_TOMBSTONED (TS 2590825fe67b48c9adb5894c322237dc not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:52286
I20240502 02:04:34.556664 24262 tablet_replica.cc:330] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc: stopping tablet replica
I20240502 02:04:34.557068 24262 raft_consensus.cc:2229] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:34.557333 24262 raft_consensus.cc:2258] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:34.559408 24262 ts_tablet_manager.cc:1875] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:34.561436 24262 ts_tablet_manager.cc:1888] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 02:04:34.561528 24262 log.cc:1196] T 1683e87385ab4050824a71508d3c3f5b P 2590825fe67b48c9adb5894c322237dc: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-5/wal/wals/1683e87385ab4050824a71508d3c3f5b
I20240502 02:04:34.562036 21281 catalog_manager.cc:4830] TS 2590825fe67b48c9adb5894c322237dc (127.7.221.70:39691): tablet 1683e87385ab4050824a71508d3c3f5b (table rebalance_test_table_0 [id=e1df1cf914834ab4bf5ac68bc944cbcc]) successfully deleted
I20240502 02:04:34.562050 22718 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b"
dest_uuid: "d590702730234f799e2a26de695bd7a2"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:42676
I20240502 02:04:34.562201 22718 raft_consensus.cc:595] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 2 LEADER]: Received request to transfer leadership
I20240502 02:04:34.562724 24197 raft_consensus.cc:1050] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0: attempting to promote NON_VOTER 2590825fe67b48c9adb5894c322237dc to VOTER
I20240502 02:04:34.563133 24197 consensus_queue.cc:237] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } }
I20240502 02:04:34.564407 22447 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:34.564406 22582 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc [term 1 LEARNER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:34.564440 21706 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 02:04:34.564795 24197 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:34.564966 23779 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:34.565109 24092 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 02:04:34.566711 24197 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:34.567140 22447 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:34.567469 21706 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:34.569196 21294 catalog_manager.cc:5484] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: config changed from index 7 to 8, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:34.569578 22582 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 2590825fe67b48c9adb5894c322237dc (127.7.221.70) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:34.572304 21439 consensus_queue.cc:237] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } }
I20240502 02:04:34.573284 21706 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 02:04:34.573297 22582 raft_consensus.cc:1261] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 02:04:34.573655 24197 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:34.573817 23779 consensus_queue.cc:1035] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:34.575143 24198 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:34.575539 22582 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P 2590825fe67b48c9adb5894c322237dc [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:34.576290 21282 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6cd9e4d44fdb40448877d5560a58c9f4 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:34.576838 21706 raft_consensus.cc:2932] T 6cd9e4d44fdb40448877d5560a58c9f4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } } }
I20240502 02:04:34.577136 21290 catalog_manager.cc:5484] T 6cd9e4d44fdb40448877d5560a58c9f4 P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: config changed from index 8 to 9, VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) evicted. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2590825fe67b48c9adb5894c322237dc" member_type: VOTER last_known_addr { host: "127.7.221.70" port: 39691 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:34.582028 22428 tablet_service.cc:1514] Processing DeleteTablet for tablet 6cd9e4d44fdb40448877d5560a58c9f4 with delete_type TABLET_DATA_TOMBSTONED (TS 5e9df19de7264d1a9ee930ee770a8701 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:47004
I20240502 02:04:34.582453 24274 tablet_replica.cc:330] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701: stopping tablet replica
I20240502 02:04:34.582743 24274 raft_consensus.cc:2229] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:34.582989 24274 raft_consensus.cc:2258] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:34.583823 24274 ts_tablet_manager.cc:1875] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:34.585398 24274 ts_tablet_manager.cc:1888] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 02:04:34.585492 24274 log.cc:1196] T 6cd9e4d44fdb40448877d5560a58c9f4 P 5e9df19de7264d1a9ee930ee770a8701: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/wals/6cd9e4d44fdb40448877d5560a58c9f4
I20240502 02:04:34.585960 21280 catalog_manager.cc:4830] TS 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69:45619): tablet 6cd9e4d44fdb40448877d5560a58c9f4 (table rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]) successfully deleted
I20240502 02:04:34.730031 21439 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "c0db5fac24d54971baf964b55e62abe4"
dest_uuid: "e72358c248634f30be5f9becf0ec0fa0"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:51890
I20240502 02:04:34.730171 21439 raft_consensus.cc:595] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:34.782511 22718 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b"
dest_uuid: "d590702730234f799e2a26de695bd7a2"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:42698
I20240502 02:04:34.782676 22718 raft_consensus.cc:595] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 2 LEADER]: Received request to transfer leadership
I20240502 02:04:34.791155 22718 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b"
dest_uuid: "d590702730234f799e2a26de695bd7a2"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:42722
I20240502 02:04:34.791273 22718 raft_consensus.cc:595] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 2 LEADER]: Received request to transfer leadership
I20240502 02:04:34.838918 24096 raft_consensus.cc:979] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: : Instructing follower 252575ab96ac4a079bab6ade63792782 to start an election
I20240502 02:04:34.839052 24096 raft_consensus.cc:1067] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 2 LEADER]: Signalling peer 252575ab96ac4a079bab6ade63792782 to start an election
I20240502 02:04:34.839570 21573 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b"
dest_uuid: "252575ab96ac4a079bab6ade63792782"
from {username='slave'} at 127.7.221.68:55549
I20240502 02:04:34.839728 21573 raft_consensus.cc:482] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 02:04:34.839790 21573 raft_consensus.cc:3037] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Advancing to term 3
I20240502 02:04:34.840575 21573 raft_consensus.cc:504] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 3 FOLLOWER]: Starting forced leader election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } }
I20240502 02:04:34.841024 21573 leader_election.cc:288] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [CANDIDATE]: Term 3 election: Requested vote from peers 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557), d590702730234f799e2a26de695bd7a2 (127.7.221.68:46455), 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69:45619)
I20240502 02:04:34.841413 21706 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b" candidate_uuid: "252575ab96ac4a079bab6ade63792782" candidate_term: 3 candidate_status { last_received { term: 2 index: 9 } } ignore_live_leader: true dest_uuid: "0983e154926c44ceb508483878dba5a0"
I20240502 02:04:34.841539 21706 raft_consensus.cc:3037] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Advancing to term 3
I20240502 02:04:34.841503 22447 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b" candidate_uuid: "252575ab96ac4a079bab6ade63792782" candidate_term: 3 candidate_status { last_received { term: 2 index: 9 } } ignore_live_leader: true dest_uuid: "5e9df19de7264d1a9ee930ee770a8701"
I20240502 02:04:34.841672 22447 raft_consensus.cc:3037] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701 [term 2 FOLLOWER]: Advancing to term 3
I20240502 02:04:34.842536 21706 raft_consensus.cc:2454] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 252575ab96ac4a079bab6ade63792782 in term 3.
I20240502 02:04:34.842778 22447 raft_consensus.cc:2454] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 252575ab96ac4a079bab6ade63792782 in term 3.
I20240502 02:04:34.843119 21505 leader_election.cc:302] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 0983e154926c44ceb508483878dba5a0, 252575ab96ac4a079bab6ade63792782, 5e9df19de7264d1a9ee930ee770a8701; no voters:
I20240502 02:04:34.843295 24245 raft_consensus.cc:2781] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 3 FOLLOWER]: Leader election won for term 3
I20240502 02:04:34.843400 24245 raft_consensus.cc:686] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 3 LEADER]: Becoming Leader. State: Replica: 252575ab96ac4a079bab6ade63792782, State: Running, Role: LEADER
I20240502 02:04:34.843564 24245 consensus_queue.cc:237] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 9, Current term: 3, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } }
I20240502 02:04:34.844697 22718 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bb5988c6c4c14f46a5e598cc1a59c99b" candidate_uuid: "252575ab96ac4a079bab6ade63792782" candidate_term: 3 candidate_status { last_received { term: 2 index: 9 } } ignore_live_leader: true dest_uuid: "d590702730234f799e2a26de695bd7a2"
I20240502 02:04:34.844833 22718 raft_consensus.cc:3032] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 2 LEADER]: Stepping down as leader of term 2
I20240502 02:04:34.844888 22718 raft_consensus.cc:729] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 2 LEADER]: Becoming Follower/Learner. State: Replica: d590702730234f799e2a26de695bd7a2, State: Running, Role: LEADER
I20240502 02:04:34.845041 22718 consensus_queue.cc:260] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 9, Current term: 2, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } }
I20240502 02:04:34.845216 22718 raft_consensus.cc:3037] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 2 FOLLOWER]: Advancing to term 3
I20240502 02:04:34.845896 22718 raft_consensus.cc:2454] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 252575ab96ac4a079bab6ade63792782 in term 3.
I20240502 02:04:34.846820 21295 catalog_manager.cc:5484] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 reported cstate change: term changed from 2 to 3, leader changed from d590702730234f799e2a26de695bd7a2 (127.7.221.68) to 252575ab96ac4a079bab6ade63792782 (127.7.221.66). New cstate: current_term: 3 leader_uuid: "252575ab96ac4a079bab6ade63792782" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:34.940735 21439 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "c0db5fac24d54971baf964b55e62abe4"
dest_uuid: "e72358c248634f30be5f9becf0ec0fa0"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:51918
I20240502 02:04:34.940868 21439 raft_consensus.cc:595] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Received request to transfer leadership
I20240502 02:04:35.005263 24198 raft_consensus.cc:979] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0: : Instructing follower 252575ab96ac4a079bab6ade63792782 to start an election
I20240502 02:04:35.005389 24198 raft_consensus.cc:1067] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Signalling peer 252575ab96ac4a079bab6ade63792782 to start an election
I20240502 02:04:35.005790 21573 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "c0db5fac24d54971baf964b55e62abe4"
dest_uuid: "252575ab96ac4a079bab6ade63792782"
from {username='slave'} at 127.7.221.65:44271
I20240502 02:04:35.005911 21573 raft_consensus.cc:482] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 02:04:35.005971 21573 raft_consensus.cc:3037] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:35.006760 21573 raft_consensus.cc:504] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } }
I20240502 02:04:35.007182 21573 leader_election.cc:288] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [CANDIDATE]: Term 2 election: Requested vote from peers e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519), 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557), 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69:45619)
I20240502 02:04:35.007496 21439 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c0db5fac24d54971baf964b55e62abe4" candidate_uuid: "252575ab96ac4a079bab6ade63792782" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "e72358c248634f30be5f9becf0ec0fa0"
I20240502 02:04:35.007642 21439 raft_consensus.cc:3032] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 02:04:35.007591 22447 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c0db5fac24d54971baf964b55e62abe4" candidate_uuid: "252575ab96ac4a079bab6ade63792782" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "5e9df19de7264d1a9ee930ee770a8701"
I20240502 02:04:35.007692 21439 raft_consensus.cc:729] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: e72358c248634f30be5f9becf0ec0fa0, State: Running, Role: LEADER
I20240502 02:04:35.007732 22447 raft_consensus.cc:3037] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:35.007856 21439 consensus_queue.cc:260] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } }
I20240502 02:04:35.008064 21439 raft_consensus.cc:3037] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:35.008559 22447 raft_consensus.cc:2454] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 252575ab96ac4a079bab6ade63792782 in term 2.
I20240502 02:04:35.008812 21439 raft_consensus.cc:2454] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 252575ab96ac4a079bab6ade63792782 in term 2.
I20240502 02:04:35.008854 21706 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c0db5fac24d54971baf964b55e62abe4" candidate_uuid: "252575ab96ac4a079bab6ade63792782" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "0983e154926c44ceb508483878dba5a0"
I20240502 02:04:35.008988 21706 raft_consensus.cc:3037] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 1 FOLLOWER]: Advancing to term 2
I20240502 02:04:35.009135 21507 leader_election.cc:302] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 252575ab96ac4a079bab6ade63792782, 5e9df19de7264d1a9ee930ee770a8701, e72358c248634f30be5f9becf0ec0fa0; no voters:
I20240502 02:04:35.009282 24245 raft_consensus.cc:2781] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 02:04:35.009358 24245 raft_consensus.cc:686] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 2 LEADER]: Becoming Leader. State: Replica: 252575ab96ac4a079bab6ade63792782, State: Running, Role: LEADER
I20240502 02:04:35.009498 24245 consensus_queue.cc:237] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } }
I20240502 02:04:35.009927 21706 raft_consensus.cc:2454] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 252575ab96ac4a079bab6ade63792782 in term 2.
I20240502 02:04:35.011224 21295 catalog_manager.cc:5484] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 reported cstate change: term changed from 1 to 2, leader changed from e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) to 252575ab96ac4a079bab6ade63792782 (127.7.221.66). New cstate: current_term: 2 leader_uuid: "252575ab96ac4a079bab6ade63792782" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:35.222637 21706 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 3 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 3 index: 10. (index mismatch)
I20240502 02:04:35.223047 24245 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 02:04:35.461416 22447 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701 [term 3 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 3 index: 10. (index mismatch)
I20240502 02:04:35.461865 24245 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 02:04:35.464643 22718 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 3 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 3 index: 10. (index mismatch)
I20240502 02:04:35.465287 24357 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 02:04:35.467821 21573 consensus_queue.cc:237] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 3.10, Last appended by leader: 9, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } }
I20240502 02:04:35.468976 22447 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701 [term 3 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 3 index: 10. Preceding OpId from leader: term: 3 index: 11. (index mismatch)
I20240502 02:04:35.469096 21706 raft_consensus.cc:1261] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 3 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 3 index: 10. Preceding OpId from leader: term: 3 index: 11. (index mismatch)
I20240502 02:04:35.469370 24357 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 02:04:35.469475 24359 consensus_queue.cc:1035] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 02:04:35.470813 24247 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 [term 3 LEADER]: Committing config change with OpId 3.11: config changed from index 9 to 11, VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:35.471208 21706 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P 0983e154926c44ceb508483878dba5a0 [term 3 FOLLOWER]: Committing config change with OpId 3.11: config changed from index 9 to 11, VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:35.471346 22447 raft_consensus.cc:2932] T bb5988c6c4c14f46a5e598cc1a59c99b P 5e9df19de7264d1a9ee930ee770a8701 [term 3 FOLLOWER]: Committing config change with OpId 3.11: config changed from index 9 to 11, VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } }
I20240502 02:04:35.472565 21280 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet bb5988c6c4c14f46a5e598cc1a59c99b with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:35.473131 21295 catalog_manager.cc:5484] T bb5988c6c4c14f46a5e598cc1a59c99b P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index 9 to 11, VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) evicted. New cstate: current_term: 3 leader_uuid: "252575ab96ac4a079bab6ade63792782" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:35.475548 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet bb5988c6c4c14f46a5e598cc1a59c99b with delete_type TABLET_DATA_TOMBSTONED (TS d590702730234f799e2a26de695bd7a2 not found in new config with opid_index 11) from {username='slave'} at 127.0.0.1:42528
I20240502 02:04:35.475984 24365 tablet_replica.cc:330] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: stopping tablet replica
I20240502 02:04:35.476234 24365 raft_consensus.cc:2229] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 3 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:35.476353 24365 raft_consensus.cc:2258] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2 [term 3 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:35.477138 24365 ts_tablet_manager.cc:1875] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:35.478824 24365 ts_tablet_manager.cc:1888] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 3.10
I20240502 02:04:35.478901 24365 log.cc:1196] T bb5988c6c4c14f46a5e598cc1a59c99b P d590702730234f799e2a26de695bd7a2: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/wals/bb5988c6c4c14f46a5e598cc1a59c99b
I20240502 02:04:35.479328 21282 catalog_manager.cc:4830] TS d590702730234f799e2a26de695bd7a2 (127.7.221.68:46455): tablet bb5988c6c4c14f46a5e598cc1a59c99b (table rebalance_test_table_2 [id=c5c742d762c44789b376f2c4c64c13b2]) successfully deleted
I20240502 02:04:35.534236 22447 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 2 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 02:04:35.534662 24357 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:35.540551 21706 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 02:04:35.540904 24357 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:35.543071 21439 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 02:04:35.543650 24357 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 02:04:35.545431 21573 consensus_queue.cc:237] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } }
I20240502 02:04:35.546563 21706 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 02:04:35.546839 22447 raft_consensus.cc:1261] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 2 FOLLOWER]: Refusing update from remote peer 252575ab96ac4a079bab6ade63792782: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 02:04:35.547034 24357 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 02:04:35.547529 24357 consensus_queue.cc:1035] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 02:04:35.548563 24247 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:35.549001 21706 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P 0983e154926c44ceb508483878dba5a0 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:35.549780 22447 raft_consensus.cc:2932] T c0db5fac24d54971baf964b55e62abe4 P 5e9df19de7264d1a9ee930ee770a8701 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } }
I20240502 02:04:35.550119 21280 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c0db5fac24d54971baf964b55e62abe4 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:35.550875 21295 catalog_manager.cc:5484] T c0db5fac24d54971baf964b55e62abe4 P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index 8 to 10, VOTER e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65) evicted. New cstate: current_term: 2 leader_uuid: "252575ab96ac4a079bab6ade63792782" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "0983e154926c44ceb508483878dba5a0" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:35.553175 21417 tablet_service.cc:1514] Processing DeleteTablet for tablet c0db5fac24d54971baf964b55e62abe4 with delete_type TABLET_DATA_TOMBSTONED (TS e72358c248634f30be5f9becf0ec0fa0 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:56444
I20240502 02:04:35.553474 24371 tablet_replica.cc:330] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0: stopping tablet replica
I20240502 02:04:35.553614 24371 raft_consensus.cc:2229] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:35.553709 24371 raft_consensus.cc:2258] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:35.554335 24371 ts_tablet_manager.cc:1875] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:35.556509 24371 ts_tablet_manager.cc:1888] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 02:04:35.556589 24371 log.cc:1196] T c0db5fac24d54971baf964b55e62abe4 P e72358c248634f30be5f9becf0ec0fa0: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/c0db5fac24d54971baf964b55e62abe4
I20240502 02:04:35.557117 21282 catalog_manager.cc:4830] TS e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519): tablet c0db5fac24d54971baf964b55e62abe4 (table rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]) successfully deleted
I20240502 02:04:35.610103 21687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 02:04:35.614907 21551 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:35.616802 22428 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:35.617261 22697 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:35.617930 22562 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:35.618135 21417 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:35.665818 21439 consensus_queue.cc:237] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } }
I20240502 02:04:35.667237 22447 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 02:04:35.667304 21573 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
W20240502 02:04:35.667389 21374 consensus_peers.cc:485] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 -> Peer d590702730234f799e2a26de695bd7a2 (127.7.221.68:46455): Couldn't send request to peer d590702730234f799e2a26de695bd7a2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a22071eab1054f0ebc97dfd4e0515148. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:35.667730 24424 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 02:04:35.667932 24422 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 02:04:35.669728 24198 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } } }
I20240502 02:04:35.671044 21573 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } } }
I20240502 02:04:35.671624 22447 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } } }
I20240502 02:04:35.672174 21295 catalog_manager.cc:5484] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 reported cstate change: config changed from index 9 to 10, NON_VOTER d590702730234f799e2a26de695bd7a2 (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:36.196336 24444 ts_tablet_manager.cc:902] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2: Initiating tablet copy from peer e72358c248634f30be5f9becf0ec0fa0 (127.7.221.65:38519)
I20240502 02:04:36.196772 24444 tablet_copy_client.cc:308] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:38519
I20240502 02:04:36.197281 21460 tablet_copy_service.cc:140] P e72358c248634f30be5f9becf0ec0fa0: Received BeginTabletCopySession request for tablet a22071eab1054f0ebc97dfd4e0515148 from peer d590702730234f799e2a26de695bd7a2 ({username='slave'} at 127.7.221.68:36261)
I20240502 02:04:36.197414 21460 tablet_copy_service.cc:161] P e72358c248634f30be5f9becf0ec0fa0: Beginning new tablet copy session on tablet a22071eab1054f0ebc97dfd4e0515148 from peer d590702730234f799e2a26de695bd7a2 at {username='slave'} at 127.7.221.68:36261: session id = d590702730234f799e2a26de695bd7a2-a22071eab1054f0ebc97dfd4e0515148
I20240502 02:04:36.198329 21460 tablet_copy_source_session.cc:205] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:36.199204 24444 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a22071eab1054f0ebc97dfd4e0515148. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:36.200883 24444 tablet_copy_client.cc:783] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:36.201107 24444 tablet_copy_client.cc:647] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:36.202191 24444 tablet_copy_client.cc:523] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:36.203155 24444 tablet_bootstrap.cc:492] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2: Bootstrap starting.
I20240502 02:04:36.221518 24444 tablet_bootstrap.cc:492] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2: Bootstrap replayed 1/1 log segments. Stats: ops{read=10 overwritten=0 applied=10 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:36.221771 24444 tablet_bootstrap.cc:492] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2: Bootstrap complete.
I20240502 02:04:36.221925 24444 ts_tablet_manager.cc:1367] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2: Time spent bootstrapping tablet: real 0.019s user 0.013s sys 0.003s
I20240502 02:04:36.222275 24444 raft_consensus.cc:348] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } }
I20240502 02:04:36.222376 24444 raft_consensus.cc:729] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d590702730234f799e2a26de695bd7a2, State: Initialized, Role: LEARNER
I20240502 02:04:36.222481 24444 consensus_queue.cc:260] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 10, Last appended: 1.10, Last appended by leader: 10, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: true } }
I20240502 02:04:36.222813 24444 ts_tablet_manager.cc:1398] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2: Time spent starting tablet: real 0.001s user 0.004s sys 0.000s
I20240502 02:04:36.223356 21460 tablet_copy_service.cc:342] P e72358c248634f30be5f9becf0ec0fa0: Request end of tablet copy session d590702730234f799e2a26de695bd7a2-a22071eab1054f0ebc97dfd4e0515148 received from {username='slave'} at 127.7.221.68:36261
I20240502 02:04:36.223443 21460 tablet_copy_service.cc:433] P e72358c248634f30be5f9becf0ec0fa0: ending tablet copy session d590702730234f799e2a26de695bd7a2-a22071eab1054f0ebc97dfd4e0515148 on tablet a22071eab1054f0ebc97dfd4e0515148 with peer d590702730234f799e2a26de695bd7a2
I20240502 02:04:36.589967 22715 raft_consensus.cc:1203] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.9->[1.10-1.10] Dedup: 1.10->[]
I20240502 02:04:37.058362 24462 raft_consensus.cc:1050] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0: attempting to promote NON_VOTER d590702730234f799e2a26de695bd7a2 to VOTER
I20240502 02:04:37.058746 24462 consensus_queue.cc:237] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } }
I20240502 02:04:37.060021 22715 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2 [term 1 LEARNER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 02:04:37.060114 22447 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 02:04:37.060290 21573 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 02:04:37.060458 24461 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 02:04:37.061439 24475 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 02:04:37.061611 24462 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 02:04:37.063524 24462 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:37.064105 22715 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:37.064146 21573 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:37.065755 22447 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:37.066814 21290 catalog_manager.cc:5484] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index 10 to 11, d590702730234f799e2a26de695bd7a2 (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "5e9df19de7264d1a9ee930ee770a8701" member_type: VOTER last_known_addr { host: "127.7.221.69" port: 45619 } attrs { promote: false replace: true } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:37.070912 21439 consensus_queue.cc:237] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } }
I20240502 02:04:37.071954 21573 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 02:04:37.072088 22715 raft_consensus.cc:1261] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Refusing update from remote peer e72358c248634f30be5f9becf0ec0fa0: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 02:04:37.072309 24461 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 02:04:37.072463 24475 consensus_queue.cc:1035] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 02:04:37.074002 24424 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P e72358c248634f30be5f9becf0ec0fa0 [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:37.074381 21573 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:37.074380 22715 raft_consensus.cc:2932] T a22071eab1054f0ebc97dfd4e0515148 P d590702730234f799e2a26de695bd7a2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:37.075500 21282 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a22071eab1054f0ebc97dfd4e0515148 with cas_config_opid_index 11: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:37.076578 21295 catalog_manager.cc:5484] T a22071eab1054f0ebc97dfd4e0515148 P 252575ab96ac4a079bab6ade63792782 reported cstate change: config changed from index 11 to 12, VOTER 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69) evicted. New cstate: current_term: 1 leader_uuid: "e72358c248634f30be5f9becf0ec0fa0" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "e72358c248634f30be5f9becf0ec0fa0" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 38519 } } peers { permanent_uuid: "252575ab96ac4a079bab6ade63792782" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 34219 } attrs { promote: false } } peers { permanent_uuid: "d590702730234f799e2a26de695bd7a2" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 46455 } attrs { promote: false } } }
I20240502 02:04:37.078578 22428 tablet_service.cc:1514] Processing DeleteTablet for tablet a22071eab1054f0ebc97dfd4e0515148 with delete_type TABLET_DATA_TOMBSTONED (TS 5e9df19de7264d1a9ee930ee770a8701 not found in new config with opid_index 12) from {username='slave'} at 127.0.0.1:47004
I20240502 02:04:37.078961 24484 tablet_replica.cc:330] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701: stopping tablet replica
I20240502 02:04:37.079068 24484 raft_consensus.cc:2229] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:37.079197 24484 raft_consensus.cc:2258] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:37.079741 24484 ts_tablet_manager.cc:1875] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:37.081298 24484 ts_tablet_manager.cc:1888] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.11
I20240502 02:04:37.081362 24484 log.cc:1196] T a22071eab1054f0ebc97dfd4e0515148 P 5e9df19de7264d1a9ee930ee770a8701: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.Rebalance_idx3_rf3_t3_l_3_3.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/wals/a22071eab1054f0ebc97dfd4e0515148
I20240502 02:04:37.081780 21280 catalog_manager.cc:4830] TS 5e9df19de7264d1a9ee930ee770a8701 (127.7.221.69:45619): tablet a22071eab1054f0ebc97dfd4e0515148 (table rebalance_test_table_1 [id=1e895fb18bec4a288dab6f46be3b887b]) successfully deleted
I20240502 02:04:37.169530 21551 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:37.170382 22562 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:37.171828 22428 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:37.180944 21687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 02:04:37.192793 21417 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:37.193271 22697 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:37.268328 21551 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:37.268790 21687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 02:04:37.269273 22562 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:37.271812 22428 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:37.279726 22697 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:37.294989 21417 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:37.325707 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 21357
I20240502 02:04:37.333313 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 21489
I20240502 02:04:37.340519 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 21624
I20240502 02:04:37.347375 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 22633
W20240502 02:04:37.348582 22518 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.67:35557: connect: Connection refused (error 111)
W20240502 02:04:37.349574 22518 consensus_peers.cc:485] T 2fd616a594f245a3b13353f1c9cb9064 P 2590825fe67b48c9adb5894c322237dc -> Peer 0983e154926c44ceb508483878dba5a0 (127.7.221.67:35557): Couldn't send request to peer 0983e154926c44ceb508483878dba5a0. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.67:35557: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:37.353544 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 22364
I20240502 02:04:37.359411 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 22498
I20240502 02:04:37.365249 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 21265
2024-05-02T02:04:37Z chronyd exiting
[ OK ] LocationAwareRebalancingParamTest.Rebalance/idx3_rf3_t3_l_3_3 (17583 ms)
[ RUN ] LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer/idx2_rf3_t3_l_3_2
I20240502 02:04:37.394248 8053 test_util.cc:268] Using random seed: 108321603
I20240502 02:04:37.395071 8053 ts_itest-base.cc:112] Starting cluster with:
I20240502 02:04:37.395109 8053 ts_itest-base.cc:113] --------------
I20240502 02:04:37.395129 8053 ts_itest-base.cc:114] 5 tablet servers
I20240502 02:04:37.395146 8053 ts_itest-base.cc:115] 3 replicas per TS
I20240502 02:04:37.395170 8053 ts_itest-base.cc:116] --------------
2024-05-02T02:04:37Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T02:04:37Z Disabled control of system clock
I20240502 02:04:37.402987 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.7.221.126:43125
--webserver_interface=127.7.221.126
--webserver_port=0
--builtin_ntp_servers=127.7.221.84:39799
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.7.221.126:43125
--location_mapping_cmd=/tmp/dist-test-tasko3zIVx/build/debug/bin/testdata/assign-location.py --state_store=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/location-assignment.state --map /A:3 --map /B:2
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false
--master_client_location_assignment_enabled=false
--allow_unsafe_replication_factor with env {}
W20240502 02:04:37.502929 24561 flags.cc:425] Enabled unsafe flag: --allow_unsafe_replication_factor=true
W20240502 02:04:37.503167 24561 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:37.503225 24561 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:37.503299 24561 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:37.506135 24561 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:37.506209 24561 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 02:04:37.506255 24561 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:37.506285 24561 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 02:04:37.506325 24561 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 02:04:37.509737 24561 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39799
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
--allow_unsafe_replication_factor=true
--enable_range_replica_placement=false
--location_mapping_cmd=/tmp/dist-test-tasko3zIVx/build/debug/bin/testdata/assign-location.py --state_store=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/location-assignment.state --map /A:3 --map /B:2
--ipki_ca_key_size=768
--master_addresses=127.7.221.126:43125
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.7.221.126:43125
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.7.221.126
--webserver_port=0
--never_fsync=true
--heap_profile_path=/tmp/kudu.24561
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:37.510547 24561 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:37.511639 24561 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:37.518945 24566 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:37.519016 24567 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:37.519652 24569 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:37.520329 24561 server_base.cc:1034] running on GCE node
I20240502 02:04:37.520773 24561 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:37.521634 24561 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:37.522797 24561 hybrid_clock.cc:648] HybridClock initialized: now 1714615477522781 us; error 31 us; skew 500 ppm
I20240502 02:04:37.524545 24561 webserver.cc:460] Webserver started at http://127.7.221.126:33655/ using document root <none> and password file <none>
I20240502 02:04:37.525061 24561 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:37.525122 24561 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:37.525285 24561 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:37.526841 24561 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "ca67a8420c7240a2b1a5402cca3cca8e"
format_stamp: "Formatted at 2024-05-02 02:04:37 on dist-test-slave-dmxt"
I20240502 02:04:37.527325 24561 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "ca67a8420c7240a2b1a5402cca3cca8e"
format_stamp: "Formatted at 2024-05-02 02:04:37 on dist-test-slave-dmxt"
I20240502 02:04:37.530539 24561 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.001s sys 0.002s
I20240502 02:04:37.532771 24575 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:37.533712 24561 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:37.533829 24561 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
uuid: "ca67a8420c7240a2b1a5402cca3cca8e"
format_stamp: "Formatted at 2024-05-02 02:04:37 on dist-test-slave-dmxt"
I20240502 02:04:37.533917 24561 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:37.564136 24561 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:37.564813 24561 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:37.564996 24561 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:37.571377 24561 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.126:43125
I20240502 02:04:37.571384 24629 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.126:43125 every 8 connection(s)
I20240502 02:04:37.572301 24561 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 02:04:37.574712 24630 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:37.577631 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 24561
I20240502 02:04:37.577764 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 02:04:37.579869 24630 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e: Bootstrap starting.
I20240502 02:04:37.581820 24630 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:37.582532 24630 log.cc:824] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:37.584211 24630 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e: No bootstrap required, opened a new log
I20240502 02:04:37.586694 24630 raft_consensus.cc:348] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ca67a8420c7240a2b1a5402cca3cca8e" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43125 } }
I20240502 02:04:37.586861 24630 raft_consensus.cc:374] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:37.586925 24630 raft_consensus.cc:729] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ca67a8420c7240a2b1a5402cca3cca8e, State: Initialized, Role: FOLLOWER
I20240502 02:04:37.587309 24630 consensus_queue.cc:260] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ca67a8420c7240a2b1a5402cca3cca8e" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43125 } }
I20240502 02:04:37.587424 24630 raft_consensus.cc:388] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 02:04:37.587493 24630 raft_consensus.cc:482] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 02:04:37.587599 24630 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:37.588562 24630 raft_consensus.cc:504] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ca67a8420c7240a2b1a5402cca3cca8e" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43125 } }
I20240502 02:04:37.588851 24630 leader_election.cc:302] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: ca67a8420c7240a2b1a5402cca3cca8e; no voters:
I20240502 02:04:37.589079 24630 leader_election.cc:288] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 02:04:37.589162 24635 raft_consensus.cc:2781] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:37.589406 24635 raft_consensus.cc:686] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [term 1 LEADER]: Becoming Leader. State: Replica: ca67a8420c7240a2b1a5402cca3cca8e, State: Running, Role: LEADER
I20240502 02:04:37.589735 24635 consensus_queue.cc:237] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ca67a8420c7240a2b1a5402cca3cca8e" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43125 } }
I20240502 02:04:37.590046 24630 sys_catalog.cc:564] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [sys.catalog]: configured and running, proceeding with master startup.
I20240502 02:04:37.591051 24637 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [sys.catalog]: SysCatalogTable state changed. Reason: New leader ca67a8420c7240a2b1a5402cca3cca8e. Latest consensus state: current_term: 1 leader_uuid: "ca67a8420c7240a2b1a5402cca3cca8e" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ca67a8420c7240a2b1a5402cca3cca8e" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43125 } } }
I20240502 02:04:37.591176 24637 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [sys.catalog]: This master's current role is: LEADER
I20240502 02:04:37.591332 24636 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "ca67a8420c7240a2b1a5402cca3cca8e" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ca67a8420c7240a2b1a5402cca3cca8e" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43125 } } }
I20240502 02:04:37.591483 24636 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e [sys.catalog]: This master's current role is: LEADER
I20240502 02:04:37.591754 24645 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 02:04:37.594455 24645 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 02:04:37.598902 24645 catalog_manager.cc:1347] Generated new cluster ID: 1067314afb18418bb9d2303264020e5e
I20240502 02:04:37.598963 24645 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 02:04:37.614538 24645 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 02:04:37.615393 24645 catalog_manager.cc:1504] Loading token signing keys...
I20240502 02:04:37.620748 24645 catalog_manager.cc:5857] T 00000000000000000000000000000000 P ca67a8420c7240a2b1a5402cca3cca8e: Generated new TSK 0
I20240502 02:04:37.621318 24645 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 02:04:37.628124 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.65:0
--local_ip_for_outbound_sockets=127.7.221.65
--webserver_interface=127.7.221.65
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43125
--builtin_ntp_servers=127.7.221.84:39799
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 02:04:37.728479 24654 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:37.728679 24654 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:37.728747 24654 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:37.731570 24654 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:37.731631 24654 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:37.731722 24654 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.65
I20240502 02:04:37.734891 24654 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39799
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.7.221.65
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43125
--never_fsync=true
--heap_profile_path=/tmp/kudu.24654
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.65
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:37.735631 24654 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:37.736675 24654 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:37.744199 24659 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:37.744814 24662 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:37.745494 24660 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:37.746141 24654 server_base.cc:1034] running on GCE node
I20240502 02:04:37.746500 24654 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:37.747054 24654 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:37.748252 24654 hybrid_clock.cc:648] HybridClock initialized: now 1714615477748217 us; error 37 us; skew 500 ppm
I20240502 02:04:37.750010 24654 webserver.cc:460] Webserver started at http://127.7.221.65:42935/ using document root <none> and password file <none>
I20240502 02:04:37.750531 24654 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:37.750620 24654 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:37.750789 24654 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:37.752298 24654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "388f78c5934b4f0caad6ea8aaee10eef"
format_stamp: "Formatted at 2024-05-02 02:04:37 on dist-test-slave-dmxt"
I20240502 02:04:37.752736 24654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "388f78c5934b4f0caad6ea8aaee10eef"
format_stamp: "Formatted at 2024-05-02 02:04:37 on dist-test-slave-dmxt"
I20240502 02:04:37.755728 24654 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.002s
I20240502 02:04:37.757580 24668 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:37.758467 24654 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:37.758566 24654 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "388f78c5934b4f0caad6ea8aaee10eef"
format_stamp: "Formatted at 2024-05-02 02:04:37 on dist-test-slave-dmxt"
I20240502 02:04:37.758677 24654 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:37.775580 24654 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:37.776185 24654 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:37.776366 24654 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:37.776880 24654 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:37.777760 24654 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:37.777812 24654 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:37.777868 24654 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:37.777901 24654 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:37.785559 24654 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.65:34793
I20240502 02:04:37.785606 24786 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.65:34793 every 8 connection(s)
I20240502 02:04:37.786492 24654 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 02:04:37.794093 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 24654
I20240502 02:04:37.794234 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 02:04:37.794571 24787 heartbeater.cc:347] Connected to a master server at 127.7.221.126:43125
I20240502 02:04:37.794822 24787 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:37.795380 24787 heartbeater.cc:511] Master 127.7.221.126:43125 requested a full tablet report, sending...
I20240502 02:04:37.796320 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.66:0
--local_ip_for_outbound_sockets=127.7.221.66
--webserver_interface=127.7.221.66
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43125
--builtin_ntp_servers=127.7.221.84:39799
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 02:04:37.831960 24592 ts_manager.cc:194] Registered new tserver with Master: 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793)
I20240502 02:04:37.833772 24592 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.65:37045
W20240502 02:04:37.898926 24790 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:37.899117 24790 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:37.899156 24790 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:37.901795 24790 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:37.901842 24790 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:37.901898 24790 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.66
I20240502 02:04:37.905153 24790 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39799
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.7.221.66
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43125
--never_fsync=true
--heap_profile_path=/tmp/kudu.24790
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.66
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:37.905997 24790 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:37.907121 24790 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:37.915230 24796 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:37.915313 24799 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:37.915879 24797 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:37.916585 24790 server_base.cc:1034] running on GCE node
I20240502 02:04:37.916975 24790 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:37.917480 24790 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:37.918669 24790 hybrid_clock.cc:648] HybridClock initialized: now 1714615477918657 us; error 23 us; skew 500 ppm
I20240502 02:04:37.920374 24790 webserver.cc:460] Webserver started at http://127.7.221.66:38613/ using document root <none> and password file <none>
I20240502 02:04:37.920893 24790 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:37.920954 24790 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:37.921147 24790 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:37.922813 24790 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "2ae58cadc2aa4eb79ee907b17ce7a049"
format_stamp: "Formatted at 2024-05-02 02:04:37 on dist-test-slave-dmxt"
I20240502 02:04:37.923272 24790 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "2ae58cadc2aa4eb79ee907b17ce7a049"
format_stamp: "Formatted at 2024-05-02 02:04:37 on dist-test-slave-dmxt"
I20240502 02:04:37.926414 24790 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 02:04:37.928488 24805 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:37.929375 24790 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:37.929474 24790 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "2ae58cadc2aa4eb79ee907b17ce7a049"
format_stamp: "Formatted at 2024-05-02 02:04:37 on dist-test-slave-dmxt"
I20240502 02:04:37.929548 24790 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:37.952287 24790 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:37.952941 24790 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:37.953099 24790 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:37.953663 24790 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:37.954735 24790 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:37.954790 24790 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:37.954843 24790 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:37.954872 24790 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:37.965083 24790 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.66:43575
I20240502 02:04:37.965124 24922 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.66:43575 every 8 connection(s)
I20240502 02:04:37.965991 24790 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 02:04:37.973062 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 24790
I20240502 02:04:37.973191 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 02:04:37.974138 24923 heartbeater.cc:347] Connected to a master server at 127.7.221.126:43125
I20240502 02:04:37.974339 24923 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:37.974949 24923 heartbeater.cc:511] Master 127.7.221.126:43125 requested a full tablet report, sending...
I20240502 02:04:37.975411 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.67:0
--local_ip_for_outbound_sockets=127.7.221.67
--webserver_interface=127.7.221.67
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43125
--builtin_ntp_servers=127.7.221.84:39799
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 02:04:38.008411 24592 ts_manager.cc:194] Registered new tserver with Master: 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66:43575)
I20240502 02:04:38.009126 24592 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.66:35279
W20240502 02:04:38.078714 24926 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:38.078915 24926 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:38.078981 24926 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:38.081722 24926 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:38.081784 24926 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:38.081893 24926 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.67
I20240502 02:04:38.085323 24926 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39799
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.7.221.67
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43125
--never_fsync=true
--heap_profile_path=/tmp/kudu.24926
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.67
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:38.086215 24926 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:38.087285 24926 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:38.094834 24933 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:38.094919 24932 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:38.095484 24935 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:38.096179 24926 server_base.cc:1034] running on GCE node
I20240502 02:04:38.096503 24926 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:38.096984 24926 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:38.098131 24926 hybrid_clock.cc:648] HybridClock initialized: now 1714615478098118 us; error 48 us; skew 500 ppm
I20240502 02:04:38.099870 24926 webserver.cc:460] Webserver started at http://127.7.221.67:38363/ using document root <none> and password file <none>
I20240502 02:04:38.100371 24926 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:38.100430 24926 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:38.100592 24926 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:38.102167 24926 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "a3dfcff51db449bcaeb54a8d97d18942"
format_stamp: "Formatted at 2024-05-02 02:04:38 on dist-test-slave-dmxt"
I20240502 02:04:38.102713 24926 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "a3dfcff51db449bcaeb54a8d97d18942"
format_stamp: "Formatted at 2024-05-02 02:04:38 on dist-test-slave-dmxt"
I20240502 02:04:38.105863 24926 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.001s
I20240502 02:04:38.107946 24942 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:38.108865 24926 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:38.108995 24926 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "a3dfcff51db449bcaeb54a8d97d18942"
format_stamp: "Formatted at 2024-05-02 02:04:38 on dist-test-slave-dmxt"
I20240502 02:04:38.109073 24926 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:38.142518 24926 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:38.143220 24926 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:38.143381 24926 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:38.143936 24926 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:38.144867 24926 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:38.144917 24926 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:38.144979 24926 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:38.145012 24926 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:38.153120 24926 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.67:43073
I20240502 02:04:38.153149 25059 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.67:43073 every 8 connection(s)
I20240502 02:04:38.154153 24926 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 02:04:38.161685 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 24926
I20240502 02:04:38.161811 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 02:04:38.163008 25060 heartbeater.cc:347] Connected to a master server at 127.7.221.126:43125
I20240502 02:04:38.163225 25060 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:38.163723 25060 heartbeater.cc:511] Master 127.7.221.126:43125 requested a full tablet report, sending...
I20240502 02:04:38.163983 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.68:0
--local_ip_for_outbound_sockets=127.7.221.68
--webserver_interface=127.7.221.68
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43125
--builtin_ntp_servers=127.7.221.84:39799
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 02:04:38.196388 24591 ts_manager.cc:194] Registered new tserver with Master: a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073)
I20240502 02:04:38.197137 24591 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.67:37189
W20240502 02:04:38.268399 25063 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:38.268673 25063 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:38.268721 25063 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:38.271673 25063 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:38.271745 25063 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:38.271823 25063 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.68
I20240502 02:04:38.275189 25063 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39799
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.68:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.7.221.68
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43125
--never_fsync=true
--heap_profile_path=/tmp/kudu.25063
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.68
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:38.276008 25063 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:38.277031 25063 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:38.284675 25071 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:38.285370 25072 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:38.286139 25074 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:38.286760 25063 server_base.cc:1034] running on GCE node
I20240502 02:04:38.287103 25063 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:38.287645 25063 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:38.288816 25063 hybrid_clock.cc:648] HybridClock initialized: now 1714615478288808 us; error 28 us; skew 500 ppm
I20240502 02:04:38.290560 25063 webserver.cc:460] Webserver started at http://127.7.221.68:36071/ using document root <none> and password file <none>
I20240502 02:04:38.291092 25063 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:38.291143 25063 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:38.291321 25063 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:38.292901 25063 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "d6365f5df381444281762b821cfdce85"
format_stamp: "Formatted at 2024-05-02 02:04:38 on dist-test-slave-dmxt"
I20240502 02:04:38.293316 25063 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "d6365f5df381444281762b821cfdce85"
format_stamp: "Formatted at 2024-05-02 02:04:38 on dist-test-slave-dmxt"
I20240502 02:04:38.296411 25063 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.003s
I20240502 02:04:38.298472 25080 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:38.299401 25063 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
I20240502 02:04:38.299492 25063 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "d6365f5df381444281762b821cfdce85"
format_stamp: "Formatted at 2024-05-02 02:04:38 on dist-test-slave-dmxt"
I20240502 02:04:38.299573 25063 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:38.319769 25063 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:38.320384 25063 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:38.320540 25063 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:38.321041 25063 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:38.321870 25063 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:38.321914 25063 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:38.321949 25063 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:38.321979 25063 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:38.330345 25063 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.68:38221
I20240502 02:04:38.330381 25195 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.68:38221 every 8 connection(s)
I20240502 02:04:38.331254 25063 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 02:04:38.339465 25197 heartbeater.cc:347] Connected to a master server at 127.7.221.126:43125
I20240502 02:04:38.339669 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 25063
I20240502 02:04:38.339696 25197 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:38.339790 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 02:04:38.340260 25197 heartbeater.cc:511] Master 127.7.221.126:43125 requested a full tablet report, sending...
I20240502 02:04:38.341943 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.69:0
--local_ip_for_outbound_sockets=127.7.221.69
--webserver_interface=127.7.221.69
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43125
--builtin_ntp_servers=127.7.221.84:39799
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 02:04:38.372396 24592 ts_manager.cc:194] Registered new tserver with Master: d6365f5df381444281762b821cfdce85 (127.7.221.68:38221)
I20240502 02:04:38.373129 24592 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.68:48913
W20240502 02:04:38.443802 25202 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:38.444031 25202 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:38.444101 25202 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:38.447234 25202 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:38.447315 25202 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:38.447407 25202 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.69
I20240502 02:04:38.451303 25202 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39799
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.69:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.7.221.69
--webserver_port=0
--tserver_master_addrs=127.7.221.126:43125
--never_fsync=true
--heap_profile_path=/tmp/kudu.25202
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.69
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:38.452056 25202 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:38.453024 25202 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:38.460606 25210 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:38.461190 25208 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:38.461665 25207 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:38.462344 25202 server_base.cc:1034] running on GCE node
I20240502 02:04:38.462730 25202 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:38.463245 25202 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:38.464375 25202 hybrid_clock.cc:648] HybridClock initialized: now 1714615478464357 us; error 29 us; skew 500 ppm
I20240502 02:04:38.466231 25202 webserver.cc:460] Webserver started at http://127.7.221.69:39651/ using document root <none> and password file <none>
I20240502 02:04:38.466805 25202 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:38.466905 25202 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:38.467087 25202 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 02:04:38.468659 25202 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "7988227e2b74402483fda5bf517f0c0e"
format_stamp: "Formatted at 2024-05-02 02:04:38 on dist-test-slave-dmxt"
I20240502 02:04:38.469105 25202 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "7988227e2b74402483fda5bf517f0c0e"
format_stamp: "Formatted at 2024-05-02 02:04:38 on dist-test-slave-dmxt"
I20240502 02:04:38.472112 25202 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 02:04:38.474066 25216 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:38.475026 25202 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:38.475121 25202 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "7988227e2b74402483fda5bf517f0c0e"
format_stamp: "Formatted at 2024-05-02 02:04:38 on dist-test-slave-dmxt"
I20240502 02:04:38.475201 25202 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:38.489195 25202 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:38.489725 25202 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:38.489900 25202 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:38.490370 25202 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:38.491319 25202 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:38.491380 25202 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:38.491423 25202 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:38.491451 25202 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:38.499321 25202 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.69:43067
I20240502 02:04:38.499348 25331 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.69:43067 every 8 connection(s)
I20240502 02:04:38.500161 25202 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 02:04:38.507508 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 25202
I20240502 02:04:38.507651 8053 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 02:04:38.508065 25332 heartbeater.cc:347] Connected to a master server at 127.7.221.126:43125
I20240502 02:04:38.508272 25332 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:38.508853 25332 heartbeater.cc:511] Master 127.7.221.126:43125 requested a full tablet report, sending...
I20240502 02:04:38.540879 24591 ts_manager.cc:194] Registered new tserver with Master: 7988227e2b74402483fda5bf517f0c0e (127.7.221.69:43067)
I20240502 02:04:38.541555 24591 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.69:54539
I20240502 02:04:38.544842 8053 external_mini_cluster.cc:930] 5 TS(s) registered with all masters
I20240502 02:04:38.553586 8053 external_mini_cluster.cc:930] 5 TS(s) registered with all masters
I20240502 02:04:38.553661 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 25063
I20240502 02:04:38.560130 8053 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu with pid 25202
I20240502 02:04:38.836274 24787 heartbeater.cc:503] Master 127.7.221.126:43125 was elected leader, sending a full tablet report...
I20240502 02:04:39.011286 24923 heartbeater.cc:503] Master 127.7.221.126:43125 was elected leader, sending a full tablet report...
I20240502 02:04:39.199558 25060 heartbeater.cc:503] Master 127.7.221.126:43125 was elected leader, sending a full tablet report...
I20240502 02:04:42.318046 24591 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:52824:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 02:04:42.321259 24591 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_0 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20240502 02:04:42.337292 24853 tablet_service.cc:1467] Processing CreateTablet for tablet db9498139b2f4f73a2e782e6669ee500 (DEFAULT_TABLE table=rebalance_test_table_0 [id=764f416bfad747f4b2b400623dbbfe73]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.337390 24993 tablet_service.cc:1467] Processing CreateTablet for tablet db9498139b2f4f73a2e782e6669ee500 (DEFAULT_TABLE table=rebalance_test_table_0 [id=764f416bfad747f4b2b400623dbbfe73]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.337285 24718 tablet_service.cc:1467] Processing CreateTablet for tablet db9498139b2f4f73a2e782e6669ee500 (DEFAULT_TABLE table=rebalance_test_table_0 [id=764f416bfad747f4b2b400623dbbfe73]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.337296 24854 tablet_service.cc:1467] Processing CreateTablet for tablet dbe35b927da442d19092ca6633a22bb5 (DEFAULT_TABLE table=rebalance_test_table_0 [id=764f416bfad747f4b2b400623dbbfe73]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.337436 24717 tablet_service.cc:1467] Processing CreateTablet for tablet 97b176df06ba407298cf17d3026d5aee (DEFAULT_TABLE table=rebalance_test_table_0 [id=764f416bfad747f4b2b400623dbbfe73]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.338279 24993 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet db9498139b2f4f73a2e782e6669ee500. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.338279 24853 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet db9498139b2f4f73a2e782e6669ee500. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.338279 24717 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 97b176df06ba407298cf17d3026d5aee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.338675 24718 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet db9498139b2f4f73a2e782e6669ee500. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.338299 24716 tablet_service.cc:1467] Processing CreateTablet for tablet dbe35b927da442d19092ca6633a22bb5 (DEFAULT_TABLE table=rebalance_test_table_0 [id=764f416bfad747f4b2b400623dbbfe73]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.338876 24716 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dbe35b927da442d19092ca6633a22bb5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.338887 24992 tablet_service.cc:1467] Processing CreateTablet for tablet 97b176df06ba407298cf17d3026d5aee (DEFAULT_TABLE table=rebalance_test_table_0 [id=764f416bfad747f4b2b400623dbbfe73]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.339097 24992 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 97b176df06ba407298cf17d3026d5aee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.340315 24854 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dbe35b927da442d19092ca6633a22bb5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.342876 24991 tablet_service.cc:1467] Processing CreateTablet for tablet dbe35b927da442d19092ca6633a22bb5 (DEFAULT_TABLE table=rebalance_test_table_0 [id=764f416bfad747f4b2b400623dbbfe73]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.343114 24991 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dbe35b927da442d19092ca6633a22bb5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.345804 25354 tablet_bootstrap.cc:492] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942: Bootstrap starting.
I20240502 02:04:42.346258 25352 tablet_bootstrap.cc:492] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef: Bootstrap starting.
I20240502 02:04:42.346370 24855 tablet_service.cc:1467] Processing CreateTablet for tablet 97b176df06ba407298cf17d3026d5aee (DEFAULT_TABLE table=rebalance_test_table_0 [id=764f416bfad747f4b2b400623dbbfe73]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.346617 24855 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 97b176df06ba407298cf17d3026d5aee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.347770 25354 tablet_bootstrap.cc:654] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.347863 25352 tablet_bootstrap.cc:654] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.348479 25352 log.cc:824] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:42.348507 25354 log.cc:824] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:42.349112 25353 tablet_bootstrap.cc:492] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049: Bootstrap starting.
I20240502 02:04:42.350435 25353 tablet_bootstrap.cc:654] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.351110 25353 log.cc:824] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:42.351889 25354 tablet_bootstrap.cc:492] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942: No bootstrap required, opened a new log
I20240502 02:04:42.352007 25352 tablet_bootstrap.cc:492] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef: No bootstrap required, opened a new log
I20240502 02:04:42.352052 25354 ts_tablet_manager.cc:1367] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.003s
I20240502 02:04:42.352192 25352 ts_tablet_manager.cc:1367] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef: Time spent bootstrapping tablet: real 0.006s user 0.004s sys 0.000s
I20240502 02:04:42.352804 25353 tablet_bootstrap.cc:492] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049: No bootstrap required, opened a new log
I20240502 02:04:42.352949 25353 ts_tablet_manager.cc:1367] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 02:04:42.355041 25353 raft_consensus.cc:348] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } }
I20240502 02:04:42.355191 25353 raft_consensus.cc:374] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.355090 25354 raft_consensus.cc:348] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.355257 25353 raft_consensus.cc:729] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ae58cadc2aa4eb79ee907b17ce7a049, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.355307 25354 raft_consensus.cc:374] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.355230 25352 raft_consensus.cc:348] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } }
I20240502 02:04:42.355363 25354 raft_consensus.cc:729] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dfcff51db449bcaeb54a8d97d18942, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.355376 25352 raft_consensus.cc:374] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.355418 25352 raft_consensus.cc:729] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 388f78c5934b4f0caad6ea8aaee10eef, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.355683 25353 consensus_queue.cc:260] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } }
I20240502 02:04:42.355770 25352 consensus_queue.cc:260] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } }
I20240502 02:04:42.355770 25354 consensus_queue.cc:260] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.356428 25353 ts_tablet_manager.cc:1398] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent starting tablet: real 0.003s user 0.004s sys 0.000s
I20240502 02:04:42.356482 25354 ts_tablet_manager.cc:1398] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942: Time spent starting tablet: real 0.004s user 0.004s sys 0.000s
I20240502 02:04:42.356482 25352 ts_tablet_manager.cc:1398] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef: Time spent starting tablet: real 0.004s user 0.004s sys 0.000s
I20240502 02:04:42.356796 25352 tablet_bootstrap.cc:492] T 97b176df06ba407298cf17d3026d5aee P 388f78c5934b4f0caad6ea8aaee10eef: Bootstrap starting.
I20240502 02:04:42.357666 25352 tablet_bootstrap.cc:654] T 97b176df06ba407298cf17d3026d5aee P 388f78c5934b4f0caad6ea8aaee10eef: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.357998 25354 tablet_bootstrap.cc:492] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942: Bootstrap starting.
I20240502 02:04:42.358919 25354 tablet_bootstrap.cc:654] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.359683 25352 tablet_bootstrap.cc:492] T 97b176df06ba407298cf17d3026d5aee P 388f78c5934b4f0caad6ea8aaee10eef: No bootstrap required, opened a new log
I20240502 02:04:42.359759 25352 ts_tablet_manager.cc:1367] T 97b176df06ba407298cf17d3026d5aee P 388f78c5934b4f0caad6ea8aaee10eef: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 02:04:42.360316 25352 raft_consensus.cc:348] T 97b176df06ba407298cf17d3026d5aee P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.360416 25352 raft_consensus.cc:374] T 97b176df06ba407298cf17d3026d5aee P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.360448 25352 raft_consensus.cc:729] T 97b176df06ba407298cf17d3026d5aee P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 388f78c5934b4f0caad6ea8aaee10eef, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.360456 25353 tablet_bootstrap.cc:492] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049: Bootstrap starting.
I20240502 02:04:42.360565 25352 consensus_queue.cc:260] T 97b176df06ba407298cf17d3026d5aee P 388f78c5934b4f0caad6ea8aaee10eef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.360833 25352 ts_tablet_manager.cc:1398] T 97b176df06ba407298cf17d3026d5aee P 388f78c5934b4f0caad6ea8aaee10eef: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:42.361008 25352 tablet_bootstrap.cc:492] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef: Bootstrap starting.
I20240502 02:04:42.361357 25353 tablet_bootstrap.cc:654] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.361369 25354 tablet_bootstrap.cc:492] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942: No bootstrap required, opened a new log
I20240502 02:04:42.361513 25354 ts_tablet_manager.cc:1367] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 02:04:42.361867 25352 tablet_bootstrap.cc:654] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.363075 25354 raft_consensus.cc:348] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } }
I20240502 02:04:42.363174 25354 raft_consensus.cc:374] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.363207 25354 raft_consensus.cc:729] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dfcff51db449bcaeb54a8d97d18942, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.363368 25354 consensus_queue.cc:260] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } }
I20240502 02:04:42.363672 25354 ts_tablet_manager.cc:1398] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:42.363827 25354 tablet_bootstrap.cc:492] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942: Bootstrap starting.
I20240502 02:04:42.364632 25354 tablet_bootstrap.cc:654] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.365612 25354 tablet_bootstrap.cc:492] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942: No bootstrap required, opened a new log
I20240502 02:04:42.365612 25352 tablet_bootstrap.cc:492] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef: No bootstrap required, opened a new log
I20240502 02:04:42.365686 25352 ts_tablet_manager.cc:1367] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 02:04:42.365734 25354 ts_tablet_manager.cc:1367] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:42.365873 25353 tablet_bootstrap.cc:492] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049: No bootstrap required, opened a new log
I20240502 02:04:42.365926 25353 ts_tablet_manager.cc:1367] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 02:04:42.366060 25352 raft_consensus.cc:348] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.366102 25354 raft_consensus.cc:348] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.366147 25352 raft_consensus.cc:374] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.366173 25354 raft_consensus.cc:374] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.366173 25352 raft_consensus.cc:729] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 388f78c5934b4f0caad6ea8aaee10eef, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.366200 25354 raft_consensus.cc:729] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dfcff51db449bcaeb54a8d97d18942, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.366230 25353 raft_consensus.cc:348] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.366288 25353 raft_consensus.cc:374] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.366314 25353 raft_consensus.cc:729] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ae58cadc2aa4eb79ee907b17ce7a049, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.366284 25352 consensus_queue.cc:260] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.366326 25354 consensus_queue.cc:260] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.366420 25353 consensus_queue.cc:260] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.366503 25352 ts_tablet_manager.cc:1398] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:42.366677 25353 ts_tablet_manager.cc:1398] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:42.366873 25353 tablet_bootstrap.cc:492] T 97b176df06ba407298cf17d3026d5aee P 2ae58cadc2aa4eb79ee907b17ce7a049: Bootstrap starting.
I20240502 02:04:42.367112 25354 ts_tablet_manager.cc:1398] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:42.367960 25353 tablet_bootstrap.cc:654] T 97b176df06ba407298cf17d3026d5aee P 2ae58cadc2aa4eb79ee907b17ce7a049: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.369072 25353 tablet_bootstrap.cc:492] T 97b176df06ba407298cf17d3026d5aee P 2ae58cadc2aa4eb79ee907b17ce7a049: No bootstrap required, opened a new log
I20240502 02:04:42.369150 25353 ts_tablet_manager.cc:1367] T 97b176df06ba407298cf17d3026d5aee P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:42.369537 25353 raft_consensus.cc:348] T 97b176df06ba407298cf17d3026d5aee P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.369640 25353 raft_consensus.cc:374] T 97b176df06ba407298cf17d3026d5aee P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.369669 25353 raft_consensus.cc:729] T 97b176df06ba407298cf17d3026d5aee P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ae58cadc2aa4eb79ee907b17ce7a049, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.369814 25353 consensus_queue.cc:260] T 97b176df06ba407298cf17d3026d5aee P 2ae58cadc2aa4eb79ee907b17ce7a049 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.370038 25353 ts_tablet_manager.cc:1398] T 97b176df06ba407298cf17d3026d5aee P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
W20240502 02:04:42.408910 25061 tablet.cc:2369] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 02:04:42.452116 25359 raft_consensus.cc:482] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:42.452318 25359 raft_consensus.cc:504] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } }
I20240502 02:04:42.453382 25359 leader_election.cc:288] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66:43575), 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793)
I20240502 02:04:42.456403 25358 raft_consensus.cc:482] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:42.456593 25358 raft_consensus.cc:504] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.457351 24875 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "db9498139b2f4f73a2e782e6669ee500" candidate_uuid: "a3dfcff51db449bcaeb54a8d97d18942" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" is_pre_election: true
I20240502 02:04:42.457552 24875 raft_consensus.cc:2454] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a3dfcff51db449bcaeb54a8d97d18942 in term 0.
I20240502 02:04:42.457640 25358 leader_election.cc:288] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793), a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073)
I20240502 02:04:42.457983 24944 leader_election.cc:302] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ae58cadc2aa4eb79ee907b17ce7a049, a3dfcff51db449bcaeb54a8d97d18942; no voters:
I20240502 02:04:42.458242 25359 raft_consensus.cc:2781] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:42.458310 25359 raft_consensus.cc:482] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:42.458351 25359 raft_consensus.cc:3037] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:42.458722 24739 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "db9498139b2f4f73a2e782e6669ee500" candidate_uuid: "a3dfcff51db449bcaeb54a8d97d18942" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef" is_pre_election: true
I20240502 02:04:42.458981 24739 raft_consensus.cc:2454] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a3dfcff51db449bcaeb54a8d97d18942 in term 0.
I20240502 02:04:42.459556 25359 raft_consensus.cc:504] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } }
I20240502 02:04:42.459890 25359 leader_election.cc:288] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [CANDIDATE]: Term 1 election: Requested vote from peers 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66:43575), 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793)
I20240502 02:04:42.460516 24875 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "db9498139b2f4f73a2e782e6669ee500" candidate_uuid: "a3dfcff51db449bcaeb54a8d97d18942" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049"
I20240502 02:04:42.460635 24875 raft_consensus.cc:3037] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:42.461721 24875 raft_consensus.cc:2454] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a3dfcff51db449bcaeb54a8d97d18942 in term 1.
I20240502 02:04:42.462069 24944 leader_election.cc:302] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ae58cadc2aa4eb79ee907b17ce7a049, a3dfcff51db449bcaeb54a8d97d18942; no voters:
I20240502 02:04:42.462334 24739 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "db9498139b2f4f73a2e782e6669ee500" candidate_uuid: "a3dfcff51db449bcaeb54a8d97d18942" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef"
I20240502 02:04:42.462453 24739 raft_consensus.cc:3037] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:42.463642 25359 raft_consensus.cc:2781] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:42.463660 24738 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dbe35b927da442d19092ca6633a22bb5" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef" is_pre_election: true
I20240502 02:04:42.463719 25014 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dbe35b927da442d19092ca6633a22bb5" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dfcff51db449bcaeb54a8d97d18942" is_pre_election: true
I20240502 02:04:42.463773 24738 raft_consensus.cc:2454] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 0.
I20240502 02:04:42.463835 25359 raft_consensus.cc:686] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 LEADER]: Becoming Leader. State: Replica: a3dfcff51db449bcaeb54a8d97d18942, State: Running, Role: LEADER
I20240502 02:04:42.463892 25014 raft_consensus.cc:2454] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 0.
I20240502 02:04:42.463964 24739 raft_consensus.cc:2454] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a3dfcff51db449bcaeb54a8d97d18942 in term 1.
I20240502 02:04:42.464114 24810 leader_election.cc:302] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ae58cadc2aa4eb79ee907b17ce7a049, 388f78c5934b4f0caad6ea8aaee10eef; no voters:
I20240502 02:04:42.464118 25359 consensus_queue.cc:237] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } }
I20240502 02:04:42.464757 25358 raft_consensus.cc:2781] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:42.464884 25358 raft_consensus.cc:482] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:42.465005 25358 raft_consensus.cc:3037] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:42.466351 25358 raft_consensus.cc:504] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.466957 25358 leader_election.cc:288] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 election: Requested vote from peers 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793), a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073)
I20240502 02:04:42.467737 24739 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dbe35b927da442d19092ca6633a22bb5" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef"
I20240502 02:04:42.467954 25014 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dbe35b927da442d19092ca6633a22bb5" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dfcff51db449bcaeb54a8d97d18942"
I20240502 02:04:42.468014 24739 raft_consensus.cc:3037] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:42.468159 25014 raft_consensus.cc:3037] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:42.469616 24739 raft_consensus.cc:2454] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 1.
I20240502 02:04:42.469907 24810 leader_election.cc:302] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ae58cadc2aa4eb79ee907b17ce7a049, 388f78c5934b4f0caad6ea8aaee10eef; no voters:
I20240502 02:04:42.469931 25014 raft_consensus.cc:2454] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 1.
I20240502 02:04:42.470180 25358 raft_consensus.cc:2781] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Leader election won for term 1
W20240502 02:04:42.470383 24924 tablet.cc:2369] T 97b176df06ba407298cf17d3026d5aee P 2ae58cadc2aa4eb79ee907b17ce7a049: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 02:04:42.470398 25358 raft_consensus.cc:686] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 LEADER]: Becoming Leader. State: Replica: 2ae58cadc2aa4eb79ee907b17ce7a049, State: Running, Role: LEADER
I20240502 02:04:42.470839 25358 consensus_queue.cc:237] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.470687 24590 catalog_manager.cc:5484] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 reported cstate change: term changed from 0 to 1, leader changed from <none> to a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67). New cstate: current_term: 1 leader_uuid: "a3dfcff51db449bcaeb54a8d97d18942" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:42.473482 24592 catalog_manager.cc:5484] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } health_report { overall_health: UNKNOWN } } }
W20240502 02:04:42.541450 24788 tablet.cc:2369] T 97b176df06ba407298cf17d3026d5aee P 388f78c5934b4f0caad6ea8aaee10eef: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 02:04:42.618274 25359 raft_consensus.cc:482] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:42.618414 25359 raft_consensus.cc:504] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.618875 25359 leader_election.cc:288] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793), 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66:43575)
I20240502 02:04:42.619339 24739 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "97b176df06ba407298cf17d3026d5aee" candidate_uuid: "a3dfcff51db449bcaeb54a8d97d18942" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef" is_pre_election: true
I20240502 02:04:42.619372 24875 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "97b176df06ba407298cf17d3026d5aee" candidate_uuid: "a3dfcff51db449bcaeb54a8d97d18942" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" is_pre_election: true
I20240502 02:04:42.619467 24739 raft_consensus.cc:2454] T 97b176df06ba407298cf17d3026d5aee P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a3dfcff51db449bcaeb54a8d97d18942 in term 0.
I20240502 02:04:42.619521 24875 raft_consensus.cc:2454] T 97b176df06ba407298cf17d3026d5aee P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a3dfcff51db449bcaeb54a8d97d18942 in term 0.
I20240502 02:04:42.619796 24947 leader_election.cc:302] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 388f78c5934b4f0caad6ea8aaee10eef, a3dfcff51db449bcaeb54a8d97d18942; no voters:
I20240502 02:04:42.619961 25359 raft_consensus.cc:2781] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:42.620025 25359 raft_consensus.cc:482] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:42.620064 25359 raft_consensus.cc:3037] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:42.620783 25359 raft_consensus.cc:504] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.621088 25359 leader_election.cc:288] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [CANDIDATE]: Term 1 election: Requested vote from peers 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793), 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66:43575)
I20240502 02:04:42.621426 24739 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "97b176df06ba407298cf17d3026d5aee" candidate_uuid: "a3dfcff51db449bcaeb54a8d97d18942" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef"
I20240502 02:04:42.621425 24875 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "97b176df06ba407298cf17d3026d5aee" candidate_uuid: "a3dfcff51db449bcaeb54a8d97d18942" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049"
I20240502 02:04:42.621536 24739 raft_consensus.cc:3037] T 97b176df06ba407298cf17d3026d5aee P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:42.621536 24875 raft_consensus.cc:3037] T 97b176df06ba407298cf17d3026d5aee P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:42.622395 24875 raft_consensus.cc:2454] T 97b176df06ba407298cf17d3026d5aee P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a3dfcff51db449bcaeb54a8d97d18942 in term 1.
I20240502 02:04:42.622395 24739 raft_consensus.cc:2454] T 97b176df06ba407298cf17d3026d5aee P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a3dfcff51db449bcaeb54a8d97d18942 in term 1.
I20240502 02:04:42.622699 24944 leader_election.cc:302] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ae58cadc2aa4eb79ee907b17ce7a049, a3dfcff51db449bcaeb54a8d97d18942; no voters:
I20240502 02:04:42.622911 25359 raft_consensus.cc:2781] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:42.623020 25359 raft_consensus.cc:686] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [term 1 LEADER]: Becoming Leader. State: Replica: a3dfcff51db449bcaeb54a8d97d18942, State: Running, Role: LEADER
I20240502 02:04:42.623157 25359 consensus_queue.cc:237] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.624735 24592 catalog_manager.cc:5484] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 reported cstate change: term changed from 0 to 1, leader changed from <none> to a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67). New cstate: current_term: 1 leader_uuid: "a3dfcff51db449bcaeb54a8d97d18942" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } health_report { overall_health: HEALTHY } } }
W20240502 02:04:42.725843 25378 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:42.726099 25378 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:42.728973 25378 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_0'
I20240502 02:04:42.804350 24738 raft_consensus.cc:1261] T 97b176df06ba407298cf17d3026d5aee P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer a3dfcff51db449bcaeb54a8d97d18942: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:42.804350 24739 raft_consensus.cc:1261] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer a3dfcff51db449bcaeb54a8d97d18942: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:42.805289 24875 raft_consensus.cc:1261] T 97b176df06ba407298cf17d3026d5aee P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Refusing update from remote peer a3dfcff51db449bcaeb54a8d97d18942: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:42.805298 24874 raft_consensus.cc:1261] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Refusing update from remote peer a3dfcff51db449bcaeb54a8d97d18942: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:42.805995 25359 consensus_queue.cc:1035] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:42.806344 25359 consensus_queue.cc:1035] T 97b176df06ba407298cf17d3026d5aee P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:42.806054 25014 raft_consensus.cc:1261] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:42.808187 25358 consensus_queue.cc:1035] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:42.809484 24739 raft_consensus.cc:1261] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:42.810528 25358 consensus_queue.cc:1035] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:42.806566 25359 consensus_queue.cc:1035] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:42.817147 25372 consensus_queue.cc:1035] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:42.820948 25408 mvcc.cc:205] Tried to move back new op lower bound from 7023065017559236608 to 7023065016177844224. Current Snapshot: MvccSnapshot[applied={T|T < 7023065017559236608}]
I20240502 02:04:42.822314 25411 mvcc.cc:205] Tried to move back new op lower bound from 7023065017560379392 to 7023065016825675776. Current Snapshot: MvccSnapshot[applied={T|T < 7023065017560379392}]
I20240502 02:04:42.831822 25409 mvcc.cc:205] Tried to move back new op lower bound from 7023065017585201152 to 7023065016202928128. Current Snapshot: MvccSnapshot[applied={T|T < 7023065017568272384}]
INSERT report
rows total: 2000
time total: 93.3205 ms
time per row: 0.0466603 ms
I20240502 02:04:42.870554 24583 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:52824:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 02:04:42.871126 24583 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_1 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20240502 02:04:42.875699 24716 tablet_service.cc:1467] Processing CreateTablet for tablet 957feb66f1af40f4a077dca804bd6097 (DEFAULT_TABLE table=rebalance_test_table_1 [id=82ee71ea745f4ac2954d9d8005e40320]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.875943 24718 tablet_service.cc:1467] Processing CreateTablet for tablet da2526bec2cd41aca7a1210289741c9f (DEFAULT_TABLE table=rebalance_test_table_1 [id=82ee71ea745f4ac2954d9d8005e40320]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.876402 24855 tablet_service.cc:1467] Processing CreateTablet for tablet 957feb66f1af40f4a077dca804bd6097 (DEFAULT_TABLE table=rebalance_test_table_1 [id=82ee71ea745f4ac2954d9d8005e40320]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.876678 24855 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 957feb66f1af40f4a077dca804bd6097. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.876631 24993 tablet_service.cc:1467] Processing CreateTablet for tablet da2526bec2cd41aca7a1210289741c9f (DEFAULT_TABLE table=rebalance_test_table_1 [id=82ee71ea745f4ac2954d9d8005e40320]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.876935 24993 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet da2526bec2cd41aca7a1210289741c9f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.877071 24854 tablet_service.cc:1467] Processing CreateTablet for tablet da2526bec2cd41aca7a1210289741c9f (DEFAULT_TABLE table=rebalance_test_table_1 [id=82ee71ea745f4ac2954d9d8005e40320]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.877260 24854 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet da2526bec2cd41aca7a1210289741c9f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.878851 24853 tablet_service.cc:1467] Processing CreateTablet for tablet daee40b8ed5446c78a730cd052c9d6b3 (DEFAULT_TABLE table=rebalance_test_table_1 [id=82ee71ea745f4ac2954d9d8005e40320]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.879065 24853 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet daee40b8ed5446c78a730cd052c9d6b3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.879137 25437 tablet_bootstrap.cc:492] T da2526bec2cd41aca7a1210289741c9f P a3dfcff51db449bcaeb54a8d97d18942: Bootstrap starting.
I20240502 02:04:42.879314 24992 tablet_service.cc:1467] Processing CreateTablet for tablet 957feb66f1af40f4a077dca804bd6097 (DEFAULT_TABLE table=rebalance_test_table_1 [id=82ee71ea745f4ac2954d9d8005e40320]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.879549 24992 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 957feb66f1af40f4a077dca804bd6097. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.880419 25437 tablet_bootstrap.cc:654] T da2526bec2cd41aca7a1210289741c9f P a3dfcff51db449bcaeb54a8d97d18942: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.876646 24991 tablet_service.cc:1467] Processing CreateTablet for tablet daee40b8ed5446c78a730cd052c9d6b3 (DEFAULT_TABLE table=rebalance_test_table_1 [id=82ee71ea745f4ac2954d9d8005e40320]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.881175 24991 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet daee40b8ed5446c78a730cd052c9d6b3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.881527 24717 tablet_service.cc:1467] Processing CreateTablet for tablet daee40b8ed5446c78a730cd052c9d6b3 (DEFAULT_TABLE table=rebalance_test_table_1 [id=82ee71ea745f4ac2954d9d8005e40320]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:42.881683 25437 tablet_bootstrap.cc:492] T da2526bec2cd41aca7a1210289741c9f P a3dfcff51db449bcaeb54a8d97d18942: No bootstrap required, opened a new log
I20240502 02:04:42.881752 25437 ts_tablet_manager.cc:1367] T da2526bec2cd41aca7a1210289741c9f P a3dfcff51db449bcaeb54a8d97d18942: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 02:04:42.876000 24716 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 957feb66f1af40f4a077dca804bd6097. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.882201 24718 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet da2526bec2cd41aca7a1210289741c9f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.882179 25437 raft_consensus.cc:348] T da2526bec2cd41aca7a1210289741c9f P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.882279 25437 raft_consensus.cc:374] T da2526bec2cd41aca7a1210289741c9f P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.882308 25437 raft_consensus.cc:729] T da2526bec2cd41aca7a1210289741c9f P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dfcff51db449bcaeb54a8d97d18942, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.882478 25437 consensus_queue.cc:260] T da2526bec2cd41aca7a1210289741c9f P a3dfcff51db449bcaeb54a8d97d18942 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.882714 24717 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet daee40b8ed5446c78a730cd052c9d6b3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:42.882822 25437 ts_tablet_manager.cc:1398] T da2526bec2cd41aca7a1210289741c9f P a3dfcff51db449bcaeb54a8d97d18942: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:42.883076 25437 tablet_bootstrap.cc:492] T 957feb66f1af40f4a077dca804bd6097 P a3dfcff51db449bcaeb54a8d97d18942: Bootstrap starting.
I20240502 02:04:42.883807 25437 tablet_bootstrap.cc:654] T 957feb66f1af40f4a077dca804bd6097 P a3dfcff51db449bcaeb54a8d97d18942: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.884348 25440 tablet_bootstrap.cc:492] T da2526bec2cd41aca7a1210289741c9f P 388f78c5934b4f0caad6ea8aaee10eef: Bootstrap starting.
I20240502 02:04:42.885118 25440 tablet_bootstrap.cc:654] T da2526bec2cd41aca7a1210289741c9f P 388f78c5934b4f0caad6ea8aaee10eef: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.886026 25437 tablet_bootstrap.cc:492] T 957feb66f1af40f4a077dca804bd6097 P a3dfcff51db449bcaeb54a8d97d18942: No bootstrap required, opened a new log
I20240502 02:04:42.886111 25437 ts_tablet_manager.cc:1367] T 957feb66f1af40f4a077dca804bd6097 P a3dfcff51db449bcaeb54a8d97d18942: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 02:04:42.886556 25437 raft_consensus.cc:348] T 957feb66f1af40f4a077dca804bd6097 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.886705 25437 raft_consensus.cc:374] T 957feb66f1af40f4a077dca804bd6097 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.886749 25437 raft_consensus.cc:729] T 957feb66f1af40f4a077dca804bd6097 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dfcff51db449bcaeb54a8d97d18942, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.886874 25437 consensus_queue.cc:260] T 957feb66f1af40f4a077dca804bd6097 P a3dfcff51db449bcaeb54a8d97d18942 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.887143 25437 ts_tablet_manager.cc:1398] T 957feb66f1af40f4a077dca804bd6097 P a3dfcff51db449bcaeb54a8d97d18942: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:42.887212 25440 tablet_bootstrap.cc:492] T da2526bec2cd41aca7a1210289741c9f P 388f78c5934b4f0caad6ea8aaee10eef: No bootstrap required, opened a new log
I20240502 02:04:42.887291 25440 ts_tablet_manager.cc:1367] T da2526bec2cd41aca7a1210289741c9f P 388f78c5934b4f0caad6ea8aaee10eef: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 02:04:42.887308 25437 tablet_bootstrap.cc:492] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942: Bootstrap starting.
I20240502 02:04:42.887456 25436 tablet_bootstrap.cc:492] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049: Bootstrap starting.
I20240502 02:04:42.887748 25440 raft_consensus.cc:348] T da2526bec2cd41aca7a1210289741c9f P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.887871 25440 raft_consensus.cc:374] T da2526bec2cd41aca7a1210289741c9f P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.887904 25440 raft_consensus.cc:729] T da2526bec2cd41aca7a1210289741c9f P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 388f78c5934b4f0caad6ea8aaee10eef, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.888052 25440 consensus_queue.cc:260] T da2526bec2cd41aca7a1210289741c9f P 388f78c5934b4f0caad6ea8aaee10eef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.888201 25437 tablet_bootstrap.cc:654] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.888373 25436 tablet_bootstrap.cc:654] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.889794 25436 tablet_bootstrap.cc:492] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049: No bootstrap required, opened a new log
I20240502 02:04:42.889782 25440 ts_tablet_manager.cc:1398] T da2526bec2cd41aca7a1210289741c9f P 388f78c5934b4f0caad6ea8aaee10eef: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:42.889874 25436 ts_tablet_manager.cc:1367] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:42.890002 25437 tablet_bootstrap.cc:492] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942: No bootstrap required, opened a new log
I20240502 02:04:42.890074 25437 ts_tablet_manager.cc:1367] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 02:04:42.890300 25436 raft_consensus.cc:348] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.890414 25436 raft_consensus.cc:374] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.890448 25436 raft_consensus.cc:729] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ae58cadc2aa4eb79ee907b17ce7a049, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.890422 25437 raft_consensus.cc:348] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.890534 25437 raft_consensus.cc:374] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.890622 25437 raft_consensus.cc:729] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dfcff51db449bcaeb54a8d97d18942, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.890571 25436 consensus_queue.cc:260] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.890738 25437 consensus_queue.cc:260] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.890905 25436 ts_tablet_manager.cc:1398] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:42.890995 25437 ts_tablet_manager.cc:1398] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 02:04:42.891077 25436 tablet_bootstrap.cc:492] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049: Bootstrap starting.
I20240502 02:04:42.891578 25440 tablet_bootstrap.cc:492] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef: Bootstrap starting.
I20240502 02:04:42.891839 25436 tablet_bootstrap.cc:654] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.892339 25440 tablet_bootstrap.cc:654] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.893216 25436 tablet_bootstrap.cc:492] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049: No bootstrap required, opened a new log
I20240502 02:04:42.893304 25436 ts_tablet_manager.cc:1367] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:42.893653 25440 tablet_bootstrap.cc:492] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef: No bootstrap required, opened a new log
I20240502 02:04:42.893796 25440 ts_tablet_manager.cc:1367] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:42.893791 25436 raft_consensus.cc:348] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.893875 25436 raft_consensus.cc:374] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.893918 25436 raft_consensus.cc:729] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ae58cadc2aa4eb79ee907b17ce7a049, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.894037 25436 consensus_queue.cc:260] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.894299 25436 ts_tablet_manager.cc:1398] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:42.894372 25440 raft_consensus.cc:348] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.894464 25436 tablet_bootstrap.cc:492] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049: Bootstrap starting.
I20240502 02:04:42.894466 25440 raft_consensus.cc:374] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.894505 25440 raft_consensus.cc:729] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 388f78c5934b4f0caad6ea8aaee10eef, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.894663 25440 consensus_queue.cc:260] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.895298 25436 tablet_bootstrap.cc:654] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.895341 25440 ts_tablet_manager.cc:1398] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:42.896307 25436 tablet_bootstrap.cc:492] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049: No bootstrap required, opened a new log
I20240502 02:04:42.896387 25436 ts_tablet_manager.cc:1367] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:42.896575 25440 tablet_bootstrap.cc:492] T 957feb66f1af40f4a077dca804bd6097 P 388f78c5934b4f0caad6ea8aaee10eef: Bootstrap starting.
I20240502 02:04:42.896796 25436 raft_consensus.cc:348] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.896888 25436 raft_consensus.cc:374] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.896917 25436 raft_consensus.cc:729] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ae58cadc2aa4eb79ee907b17ce7a049, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.897059 25436 consensus_queue.cc:260] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.897392 25436 ts_tablet_manager.cc:1398] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:42.897503 25440 tablet_bootstrap.cc:654] T 957feb66f1af40f4a077dca804bd6097 P 388f78c5934b4f0caad6ea8aaee10eef: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:42.898903 25440 tablet_bootstrap.cc:492] T 957feb66f1af40f4a077dca804bd6097 P 388f78c5934b4f0caad6ea8aaee10eef: No bootstrap required, opened a new log
I20240502 02:04:42.898968 25440 ts_tablet_manager.cc:1367] T 957feb66f1af40f4a077dca804bd6097 P 388f78c5934b4f0caad6ea8aaee10eef: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:42.899289 25440 raft_consensus.cc:348] T 957feb66f1af40f4a077dca804bd6097 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.899384 25440 raft_consensus.cc:374] T 957feb66f1af40f4a077dca804bd6097 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:42.899403 25440 raft_consensus.cc:729] T 957feb66f1af40f4a077dca804bd6097 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 388f78c5934b4f0caad6ea8aaee10eef, State: Initialized, Role: FOLLOWER
I20240502 02:04:42.899534 25440 consensus_queue.cc:260] T 957feb66f1af40f4a077dca804bd6097 P 388f78c5934b4f0caad6ea8aaee10eef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.899798 25440 ts_tablet_manager.cc:1398] T 957feb66f1af40f4a077dca804bd6097 P 388f78c5934b4f0caad6ea8aaee10eef: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:42.946863 25415 raft_consensus.cc:482] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:42.946990 25415 raft_consensus.cc:504] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.947345 25415 leader_election.cc:288] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66:43575), 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793)
I20240502 02:04:42.947786 24875 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "daee40b8ed5446c78a730cd052c9d6b3" candidate_uuid: "a3dfcff51db449bcaeb54a8d97d18942" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" is_pre_election: true
I20240502 02:04:42.947810 24738 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "daee40b8ed5446c78a730cd052c9d6b3" candidate_uuid: "a3dfcff51db449bcaeb54a8d97d18942" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef" is_pre_election: true
I20240502 02:04:42.947944 24738 raft_consensus.cc:2454] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a3dfcff51db449bcaeb54a8d97d18942 in term 0.
I20240502 02:04:42.947959 24875 raft_consensus.cc:2454] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a3dfcff51db449bcaeb54a8d97d18942 in term 0.
I20240502 02:04:42.948294 24947 leader_election.cc:302] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 388f78c5934b4f0caad6ea8aaee10eef, a3dfcff51db449bcaeb54a8d97d18942; no voters:
I20240502 02:04:42.948520 25415 raft_consensus.cc:2781] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:42.948585 25415 raft_consensus.cc:482] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:42.948621 25415 raft_consensus.cc:3037] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:42.949568 25415 raft_consensus.cc:504] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.949849 25415 leader_election.cc:288] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [CANDIDATE]: Term 1 election: Requested vote from peers 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66:43575), 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793)
I20240502 02:04:42.950186 24875 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "daee40b8ed5446c78a730cd052c9d6b3" candidate_uuid: "a3dfcff51db449bcaeb54a8d97d18942" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049"
I20240502 02:04:42.950217 24738 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "daee40b8ed5446c78a730cd052c9d6b3" candidate_uuid: "a3dfcff51db449bcaeb54a8d97d18942" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef"
I20240502 02:04:42.950306 24875 raft_consensus.cc:3037] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:42.950318 24738 raft_consensus.cc:3037] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:42.951294 24875 raft_consensus.cc:2454] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a3dfcff51db449bcaeb54a8d97d18942 in term 1.
I20240502 02:04:42.951294 24738 raft_consensus.cc:2454] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a3dfcff51db449bcaeb54a8d97d18942 in term 1.
I20240502 02:04:42.951601 24947 leader_election.cc:302] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 388f78c5934b4f0caad6ea8aaee10eef, a3dfcff51db449bcaeb54a8d97d18942; no voters:
I20240502 02:04:42.951769 25415 raft_consensus.cc:2781] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:42.951866 25415 raft_consensus.cc:686] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 LEADER]: Becoming Leader. State: Replica: a3dfcff51db449bcaeb54a8d97d18942, State: Running, Role: LEADER
I20240502 02:04:42.951997 25415 consensus_queue.cc:237] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.953631 24584 catalog_manager.cc:5484] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 reported cstate change: term changed from 0 to 1, leader changed from <none> to a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67). New cstate: current_term: 1 leader_uuid: "a3dfcff51db449bcaeb54a8d97d18942" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } health_report { overall_health: HEALTHY } } }
I20240502 02:04:42.998127 25358 raft_consensus.cc:482] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:42.998247 25358 raft_consensus.cc:504] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:42.998744 25358 leader_election.cc:288] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793), a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073)
I20240502 02:04:42.999171 24738 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "957feb66f1af40f4a077dca804bd6097" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef" is_pre_election: true
I20240502 02:04:42.999208 25014 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "957feb66f1af40f4a077dca804bd6097" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dfcff51db449bcaeb54a8d97d18942" is_pre_election: true
I20240502 02:04:42.999315 24738 raft_consensus.cc:2454] T 957feb66f1af40f4a077dca804bd6097 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 0.
I20240502 02:04:42.999325 25014 raft_consensus.cc:2454] T 957feb66f1af40f4a077dca804bd6097 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 0.
I20240502 02:04:42.999644 24806 leader_election.cc:302] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ae58cadc2aa4eb79ee907b17ce7a049, a3dfcff51db449bcaeb54a8d97d18942; no voters:
I20240502 02:04:42.999809 25358 raft_consensus.cc:2781] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:42.999876 25358 raft_consensus.cc:482] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:42.999904 25358 raft_consensus.cc:3037] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:43.000697 25358 raft_consensus.cc:504] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.000969 25358 leader_election.cc:288] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 election: Requested vote from peers 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793), a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073)
I20240502 02:04:43.001263 24738 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "957feb66f1af40f4a077dca804bd6097" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef"
I20240502 02:04:43.001286 25014 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "957feb66f1af40f4a077dca804bd6097" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dfcff51db449bcaeb54a8d97d18942"
I20240502 02:04:43.001361 24738 raft_consensus.cc:3037] T 957feb66f1af40f4a077dca804bd6097 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:43.001375 25014 raft_consensus.cc:3037] T 957feb66f1af40f4a077dca804bd6097 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:43.002053 24738 raft_consensus.cc:2454] T 957feb66f1af40f4a077dca804bd6097 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 1.
I20240502 02:04:43.002136 25014 raft_consensus.cc:2454] T 957feb66f1af40f4a077dca804bd6097 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 1.
I20240502 02:04:43.002305 24810 leader_election.cc:302] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ae58cadc2aa4eb79ee907b17ce7a049, 388f78c5934b4f0caad6ea8aaee10eef; no voters:
I20240502 02:04:43.002508 25358 raft_consensus.cc:2781] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:43.002584 25358 raft_consensus.cc:686] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 LEADER]: Becoming Leader. State: Replica: 2ae58cadc2aa4eb79ee907b17ce7a049, State: Running, Role: LEADER
I20240502 02:04:43.002746 25358 consensus_queue.cc:237] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.004493 24584 catalog_manager.cc:5484] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:43.004720 25358 raft_consensus.cc:482] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:43.004801 25358 raft_consensus.cc:504] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.005080 25358 leader_election.cc:288] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793), a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073)
I20240502 02:04:43.005390 24738 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "da2526bec2cd41aca7a1210289741c9f" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef" is_pre_election: true
I20240502 02:04:43.005504 24738 raft_consensus.cc:2454] T da2526bec2cd41aca7a1210289741c9f P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 0.
I20240502 02:04:43.005499 25014 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "da2526bec2cd41aca7a1210289741c9f" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dfcff51db449bcaeb54a8d97d18942" is_pre_election: true
I20240502 02:04:43.005602 25014 raft_consensus.cc:2454] T da2526bec2cd41aca7a1210289741c9f P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 0.
I20240502 02:04:43.005738 24810 leader_election.cc:302] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ae58cadc2aa4eb79ee907b17ce7a049, 388f78c5934b4f0caad6ea8aaee10eef; no voters:
I20240502 02:04:43.005935 25358 raft_consensus.cc:2781] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:43.005997 25358 raft_consensus.cc:482] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:43.006042 25358 raft_consensus.cc:3037] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:43.006767 25358 raft_consensus.cc:504] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.007088 25358 leader_election.cc:288] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 election: Requested vote from peers 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793), a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073)
I20240502 02:04:43.007377 24738 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "da2526bec2cd41aca7a1210289741c9f" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef"
I20240502 02:04:43.007373 25014 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "da2526bec2cd41aca7a1210289741c9f" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dfcff51db449bcaeb54a8d97d18942"
I20240502 02:04:43.007467 25014 raft_consensus.cc:3037] T da2526bec2cd41aca7a1210289741c9f P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:43.007465 24738 raft_consensus.cc:3037] T da2526bec2cd41aca7a1210289741c9f P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:43.008319 25014 raft_consensus.cc:2454] T da2526bec2cd41aca7a1210289741c9f P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 1.
I20240502 02:04:43.008441 24738 raft_consensus.cc:2454] T da2526bec2cd41aca7a1210289741c9f P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 1.
I20240502 02:04:43.008664 24806 leader_election.cc:302] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ae58cadc2aa4eb79ee907b17ce7a049, a3dfcff51db449bcaeb54a8d97d18942; no voters:
I20240502 02:04:43.008864 25358 raft_consensus.cc:2781] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:43.008937 25358 raft_consensus.cc:686] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 LEADER]: Becoming Leader. State: Replica: 2ae58cadc2aa4eb79ee907b17ce7a049, State: Running, Role: LEADER
I20240502 02:04:43.009034 25358 consensus_queue.cc:237] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.010252 24584 catalog_manager.cc:5484] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } health_report { overall_health: UNKNOWN } } }
W20240502 02:04:43.128916 25451 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:43.129154 25451 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:43.132040 25451 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_1'
I20240502 02:04:43.203289 25014 raft_consensus.cc:1261] T da2526bec2cd41aca7a1210289741c9f P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:43.203908 25358 consensus_queue.cc:1035] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:43.205054 24738 raft_consensus.cc:1261] T da2526bec2cd41aca7a1210289741c9f P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:43.206279 24738 raft_consensus.cc:1261] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer a3dfcff51db449bcaeb54a8d97d18942: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:43.206887 25415 consensus_queue.cc:1035] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:43.207032 24875 raft_consensus.cc:1261] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Refusing update from remote peer a3dfcff51db449bcaeb54a8d97d18942: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:43.209460 25375 consensus_queue.cc:1035] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:43.210816 25014 raft_consensus.cc:1261] T 957feb66f1af40f4a077dca804bd6097 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:43.211213 25375 consensus_queue.cc:1035] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:43.211481 25415 consensus_queue.cc:1035] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:43.229049 24737 raft_consensus.cc:1261] T 957feb66f1af40f4a077dca804bd6097 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:43.229496 25375 consensus_queue.cc:1035] T 957feb66f1af40f4a077dca804bd6097 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 97.4366 ms
time per row: 0.0487183 ms
I20240502 02:04:43.269938 24584 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:52824:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 02:04:43.270404 24584 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_2 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20240502 02:04:43.275403 24716 tablet_service.cc:1467] Processing CreateTablet for tablet 4622b7cd6aff4ddeac39b772561004f9 (DEFAULT_TABLE table=rebalance_test_table_2 [id=b948bf193f204440a46af2aac2295c6e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:43.275460 24854 tablet_service.cc:1467] Processing CreateTablet for tablet 4622b7cd6aff4ddeac39b772561004f9 (DEFAULT_TABLE table=rebalance_test_table_2 [id=b948bf193f204440a46af2aac2295c6e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:43.275720 24716 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4622b7cd6aff4ddeac39b772561004f9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:43.275720 24854 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4622b7cd6aff4ddeac39b772561004f9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:43.276046 24992 tablet_service.cc:1467] Processing CreateTablet for tablet e5d1e27205184c7db5678c4fe61c3512 (DEFAULT_TABLE table=rebalance_test_table_2 [id=b948bf193f204440a46af2aac2295c6e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:43.276286 24992 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e5d1e27205184c7db5678c4fe61c3512. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:43.277838 25437 tablet_bootstrap.cc:492] T e5d1e27205184c7db5678c4fe61c3512 P a3dfcff51db449bcaeb54a8d97d18942: Bootstrap starting.
I20240502 02:04:43.277822 25440 tablet_bootstrap.cc:492] T 4622b7cd6aff4ddeac39b772561004f9 P 388f78c5934b4f0caad6ea8aaee10eef: Bootstrap starting.
I20240502 02:04:43.277964 24991 tablet_service.cc:1467] Processing CreateTablet for tablet 4622b7cd6aff4ddeac39b772561004f9 (DEFAULT_TABLE table=rebalance_test_table_2 [id=b948bf193f204440a46af2aac2295c6e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:43.278142 24991 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4622b7cd6aff4ddeac39b772561004f9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:43.278728 25437 tablet_bootstrap.cc:654] T e5d1e27205184c7db5678c4fe61c3512 P a3dfcff51db449bcaeb54a8d97d18942: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:43.278728 25440 tablet_bootstrap.cc:654] T 4622b7cd6aff4ddeac39b772561004f9 P 388f78c5934b4f0caad6ea8aaee10eef: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:43.279613 25436 tablet_bootstrap.cc:492] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049: Bootstrap starting.
I20240502 02:04:43.280174 25437 tablet_bootstrap.cc:492] T e5d1e27205184c7db5678c4fe61c3512 P a3dfcff51db449bcaeb54a8d97d18942: No bootstrap required, opened a new log
I20240502 02:04:43.280239 25437 ts_tablet_manager.cc:1367] T e5d1e27205184c7db5678c4fe61c3512 P a3dfcff51db449bcaeb54a8d97d18942: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 02:04:43.280238 24716 tablet_service.cc:1467] Processing CreateTablet for tablet e5d1e27205184c7db5678c4fe61c3512 (DEFAULT_TABLE table=rebalance_test_table_2 [id=b948bf193f204440a46af2aac2295c6e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:43.280238 24717 tablet_service.cc:1467] Processing CreateTablet for tablet f2ad58ac3a054d62b2ce1729ca6979fa (DEFAULT_TABLE table=rebalance_test_table_2 [id=b948bf193f204440a46af2aac2295c6e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:43.280411 25436 tablet_bootstrap.cc:654] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:43.280506 24716 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e5d1e27205184c7db5678c4fe61c3512. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:43.280601 24717 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f2ad58ac3a054d62b2ce1729ca6979fa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:43.280685 25437 raft_consensus.cc:348] T e5d1e27205184c7db5678c4fe61c3512 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.280776 25437 raft_consensus.cc:374] T e5d1e27205184c7db5678c4fe61c3512 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:43.280807 25437 raft_consensus.cc:729] T e5d1e27205184c7db5678c4fe61c3512 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dfcff51db449bcaeb54a8d97d18942, State: Initialized, Role: FOLLOWER
I20240502 02:04:43.280934 25437 consensus_queue.cc:260] T e5d1e27205184c7db5678c4fe61c3512 P a3dfcff51db449bcaeb54a8d97d18942 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.281142 24991 tablet_service.cc:1467] Processing CreateTablet for tablet f2ad58ac3a054d62b2ce1729ca6979fa (DEFAULT_TABLE table=rebalance_test_table_2 [id=b948bf193f204440a46af2aac2295c6e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:43.281209 25437 ts_tablet_manager.cc:1398] T e5d1e27205184c7db5678c4fe61c3512 P a3dfcff51db449bcaeb54a8d97d18942: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 02:04:43.281351 24991 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f2ad58ac3a054d62b2ce1729ca6979fa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:43.281355 25437 tablet_bootstrap.cc:492] T 4622b7cd6aff4ddeac39b772561004f9 P a3dfcff51db449bcaeb54a8d97d18942: Bootstrap starting.
I20240502 02:04:43.282478 24853 tablet_service.cc:1467] Processing CreateTablet for tablet f2ad58ac3a054d62b2ce1729ca6979fa (DEFAULT_TABLE table=rebalance_test_table_2 [id=b948bf193f204440a46af2aac2295c6e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:43.282712 25437 tablet_bootstrap.cc:654] T 4622b7cd6aff4ddeac39b772561004f9 P a3dfcff51db449bcaeb54a8d97d18942: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:43.282815 24853 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f2ad58ac3a054d62b2ce1729ca6979fa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:43.283672 25436 tablet_bootstrap.cc:492] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049: No bootstrap required, opened a new log
I20240502 02:04:43.283772 25436 ts_tablet_manager.cc:1367] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 02:04:43.284250 24854 tablet_service.cc:1467] Processing CreateTablet for tablet e5d1e27205184c7db5678c4fe61c3512 (DEFAULT_TABLE table=rebalance_test_table_2 [id=b948bf193f204440a46af2aac2295c6e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 02:04:43.284478 24854 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e5d1e27205184c7db5678c4fe61c3512. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:43.284266 25436 raft_consensus.cc:348] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.284760 25436 raft_consensus.cc:374] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:43.284797 25436 raft_consensus.cc:729] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ae58cadc2aa4eb79ee907b17ce7a049, State: Initialized, Role: FOLLOWER
I20240502 02:04:43.284881 25437 tablet_bootstrap.cc:492] T 4622b7cd6aff4ddeac39b772561004f9 P a3dfcff51db449bcaeb54a8d97d18942: No bootstrap required, opened a new log
I20240502 02:04:43.284950 25437 ts_tablet_manager.cc:1367] T 4622b7cd6aff4ddeac39b772561004f9 P a3dfcff51db449bcaeb54a8d97d18942: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.001s
I20240502 02:04:43.284919 25436 consensus_queue.cc:260] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.285190 25436 ts_tablet_manager.cc:1398] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:43.285341 25436 tablet_bootstrap.cc:492] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049: Bootstrap starting.
I20240502 02:04:43.285364 25437 raft_consensus.cc:348] T 4622b7cd6aff4ddeac39b772561004f9 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.285444 25437 raft_consensus.cc:374] T 4622b7cd6aff4ddeac39b772561004f9 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:43.285472 25437 raft_consensus.cc:729] T 4622b7cd6aff4ddeac39b772561004f9 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dfcff51db449bcaeb54a8d97d18942, State: Initialized, Role: FOLLOWER
I20240502 02:04:43.285589 25437 consensus_queue.cc:260] T 4622b7cd6aff4ddeac39b772561004f9 P a3dfcff51db449bcaeb54a8d97d18942 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.285885 25437 ts_tablet_manager.cc:1398] T 4622b7cd6aff4ddeac39b772561004f9 P a3dfcff51db449bcaeb54a8d97d18942: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:43.286051 25437 tablet_bootstrap.cc:492] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942: Bootstrap starting.
I20240502 02:04:43.286237 25436 tablet_bootstrap.cc:654] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:43.286963 25437 tablet_bootstrap.cc:654] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:43.287570 25436 tablet_bootstrap.cc:492] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049: No bootstrap required, opened a new log
I20240502 02:04:43.287676 25436 ts_tablet_manager.cc:1367] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:43.288123 25437 tablet_bootstrap.cc:492] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942: No bootstrap required, opened a new log
I20240502 02:04:43.288213 25437 ts_tablet_manager.cc:1367] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:43.288322 25436 raft_consensus.cc:348] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.288416 25436 raft_consensus.cc:374] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:43.288451 25436 raft_consensus.cc:729] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ae58cadc2aa4eb79ee907b17ce7a049, State: Initialized, Role: FOLLOWER
I20240502 02:04:43.288713 25436 consensus_queue.cc:260] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.288933 25440 tablet_bootstrap.cc:492] T 4622b7cd6aff4ddeac39b772561004f9 P 388f78c5934b4f0caad6ea8aaee10eef: No bootstrap required, opened a new log
I20240502 02:04:43.288908 25437 raft_consensus.cc:348] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.288992 25437 raft_consensus.cc:374] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:43.289012 25440 ts_tablet_manager.cc:1367] T 4622b7cd6aff4ddeac39b772561004f9 P 388f78c5934b4f0caad6ea8aaee10eef: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.002s
I20240502 02:04:43.289204 25436 ts_tablet_manager.cc:1398] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:43.289383 25436 tablet_bootstrap.cc:492] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049: Bootstrap starting.
I20240502 02:04:43.289414 25440 raft_consensus.cc:348] T 4622b7cd6aff4ddeac39b772561004f9 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.289510 25440 raft_consensus.cc:374] T 4622b7cd6aff4ddeac39b772561004f9 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:43.289542 25440 raft_consensus.cc:729] T 4622b7cd6aff4ddeac39b772561004f9 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 388f78c5934b4f0caad6ea8aaee10eef, State: Initialized, Role: FOLLOWER
I20240502 02:04:43.289664 25440 consensus_queue.cc:260] T 4622b7cd6aff4ddeac39b772561004f9 P 388f78c5934b4f0caad6ea8aaee10eef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.289038 25437 raft_consensus.cc:729] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dfcff51db449bcaeb54a8d97d18942, State: Initialized, Role: FOLLOWER
I20240502 02:04:43.289971 25440 ts_tablet_manager.cc:1398] T 4622b7cd6aff4ddeac39b772561004f9 P 388f78c5934b4f0caad6ea8aaee10eef: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 02:04:43.289978 25437 consensus_queue.cc:260] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.290153 25440 tablet_bootstrap.cc:492] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef: Bootstrap starting.
I20240502 02:04:43.290295 25437 ts_tablet_manager.cc:1398] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:43.291072 25436 tablet_bootstrap.cc:654] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:43.291072 25440 tablet_bootstrap.cc:654] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:43.292732 25440 tablet_bootstrap.cc:492] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef: No bootstrap required, opened a new log
I20240502 02:04:43.292826 25440 ts_tablet_manager.cc:1367] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.001s
I20240502 02:04:43.293280 25440 raft_consensus.cc:348] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.293382 25440 raft_consensus.cc:374] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:43.293421 25440 raft_consensus.cc:729] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 388f78c5934b4f0caad6ea8aaee10eef, State: Initialized, Role: FOLLOWER
I20240502 02:04:43.293536 25440 consensus_queue.cc:260] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.293648 25436 tablet_bootstrap.cc:492] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049: No bootstrap required, opened a new log
I20240502 02:04:43.293720 25436 ts_tablet_manager.cc:1367] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.001s
I20240502 02:04:43.293805 25440 ts_tablet_manager.cc:1398] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 02:04:43.294034 25440 tablet_bootstrap.cc:492] T e5d1e27205184c7db5678c4fe61c3512 P 388f78c5934b4f0caad6ea8aaee10eef: Bootstrap starting.
I20240502 02:04:43.294085 25436 raft_consensus.cc:348] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.294180 25436 raft_consensus.cc:374] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:43.294219 25436 raft_consensus.cc:729] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ae58cadc2aa4eb79ee907b17ce7a049, State: Initialized, Role: FOLLOWER
I20240502 02:04:43.294811 25440 tablet_bootstrap.cc:654] T e5d1e27205184c7db5678c4fe61c3512 P 388f78c5934b4f0caad6ea8aaee10eef: Neither blocks nor log segments found. Creating new log.
I20240502 02:04:43.294893 25436 consensus_queue.cc:260] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.295254 25436 ts_tablet_manager.cc:1398] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 02:04:43.295928 25440 tablet_bootstrap.cc:492] T e5d1e27205184c7db5678c4fe61c3512 P 388f78c5934b4f0caad6ea8aaee10eef: No bootstrap required, opened a new log
I20240502 02:04:43.296026 25440 ts_tablet_manager.cc:1367] T e5d1e27205184c7db5678c4fe61c3512 P 388f78c5934b4f0caad6ea8aaee10eef: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:43.296411 25440 raft_consensus.cc:348] T e5d1e27205184c7db5678c4fe61c3512 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.296490 25440 raft_consensus.cc:374] T e5d1e27205184c7db5678c4fe61c3512 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 02:04:43.296521 25440 raft_consensus.cc:729] T e5d1e27205184c7db5678c4fe61c3512 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 388f78c5934b4f0caad6ea8aaee10eef, State: Initialized, Role: FOLLOWER
I20240502 02:04:43.296627 25440 consensus_queue.cc:260] T e5d1e27205184c7db5678c4fe61c3512 P 388f78c5934b4f0caad6ea8aaee10eef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.296907 25440 ts_tablet_manager.cc:1398] T e5d1e27205184c7db5678c4fe61c3512 P 388f78c5934b4f0caad6ea8aaee10eef: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:43.307176 25484 raft_consensus.cc:482] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:43.307273 25484 raft_consensus.cc:504] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.307657 25484 leader_election.cc:288] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793), a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073)
I20240502 02:04:43.307988 24738 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e5d1e27205184c7db5678c4fe61c3512" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef" is_pre_election: true
I20240502 02:04:43.308053 25014 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e5d1e27205184c7db5678c4fe61c3512" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dfcff51db449bcaeb54a8d97d18942" is_pre_election: true
I20240502 02:04:43.308120 24738 raft_consensus.cc:2454] T e5d1e27205184c7db5678c4fe61c3512 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 0.
I20240502 02:04:43.308179 25014 raft_consensus.cc:2454] T e5d1e27205184c7db5678c4fe61c3512 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 0.
I20240502 02:04:43.308408 24810 leader_election.cc:302] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ae58cadc2aa4eb79ee907b17ce7a049, 388f78c5934b4f0caad6ea8aaee10eef; no voters:
I20240502 02:04:43.308557 25484 raft_consensus.cc:2781] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:43.308619 25484 raft_consensus.cc:482] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:43.308668 25484 raft_consensus.cc:3037] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:43.309386 25484 raft_consensus.cc:504] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.309677 25484 leader_election.cc:288] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 election: Requested vote from peers 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793), a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073)
I20240502 02:04:43.310021 25014 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e5d1e27205184c7db5678c4fe61c3512" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dfcff51db449bcaeb54a8d97d18942"
I20240502 02:04:43.310009 24738 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e5d1e27205184c7db5678c4fe61c3512" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef"
I20240502 02:04:43.310117 24738 raft_consensus.cc:3037] T e5d1e27205184c7db5678c4fe61c3512 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:43.310117 25014 raft_consensus.cc:3037] T e5d1e27205184c7db5678c4fe61c3512 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:43.310997 24738 raft_consensus.cc:2454] T e5d1e27205184c7db5678c4fe61c3512 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 1.
I20240502 02:04:43.310997 25014 raft_consensus.cc:2454] T e5d1e27205184c7db5678c4fe61c3512 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 1.
I20240502 02:04:43.311267 24806 leader_election.cc:302] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ae58cadc2aa4eb79ee907b17ce7a049, a3dfcff51db449bcaeb54a8d97d18942; no voters:
I20240502 02:04:43.311451 25484 raft_consensus.cc:2781] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:43.311535 25484 raft_consensus.cc:686] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 LEADER]: Becoming Leader. State: Replica: 2ae58cadc2aa4eb79ee907b17ce7a049, State: Running, Role: LEADER
I20240502 02:04:43.311683 25484 consensus_queue.cc:237] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.313082 24590 catalog_manager.cc:5484] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:43.465629 25484 raft_consensus.cc:482] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:43.465781 25484 raft_consensus.cc:504] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.466207 25484 leader_election.cc:288] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793), a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073)
I20240502 02:04:43.466679 24738 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4622b7cd6aff4ddeac39b772561004f9" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef" is_pre_election: true
I20240502 02:04:43.466689 25014 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4622b7cd6aff4ddeac39b772561004f9" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dfcff51db449bcaeb54a8d97d18942" is_pre_election: true
I20240502 02:04:43.466807 24738 raft_consensus.cc:2454] T 4622b7cd6aff4ddeac39b772561004f9 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 0.
I20240502 02:04:43.466833 25014 raft_consensus.cc:2454] T 4622b7cd6aff4ddeac39b772561004f9 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 0.
I20240502 02:04:43.467151 24810 leader_election.cc:302] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ae58cadc2aa4eb79ee907b17ce7a049, 388f78c5934b4f0caad6ea8aaee10eef; no voters:
I20240502 02:04:43.467306 25484 raft_consensus.cc:2781] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:43.467298 25486 raft_consensus.cc:482] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 02:04:43.467350 25484 raft_consensus.cc:482] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:43.467386 25484 raft_consensus.cc:3037] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:43.467381 25486 raft_consensus.cc:504] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.468237 25484 raft_consensus.cc:504] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.468364 25486 leader_election.cc:288] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66:43575), a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073)
I20240502 02:04:43.468590 25484 leader_election.cc:288] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 election: Requested vote from peers 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793), a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073)
I20240502 02:04:43.469287 25014 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4622b7cd6aff4ddeac39b772561004f9" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dfcff51db449bcaeb54a8d97d18942"
I20240502 02:04:43.469395 25014 raft_consensus.cc:3037] T 4622b7cd6aff4ddeac39b772561004f9 P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:43.470290 25014 raft_consensus.cc:2454] T 4622b7cd6aff4ddeac39b772561004f9 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 1.
I20240502 02:04:43.470643 24806 leader_election.cc:302] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ae58cadc2aa4eb79ee907b17ce7a049, a3dfcff51db449bcaeb54a8d97d18942; no voters:
I20240502 02:04:43.470731 24738 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4622b7cd6aff4ddeac39b772561004f9" candidate_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "388f78c5934b4f0caad6ea8aaee10eef"
I20240502 02:04:43.470868 25484 raft_consensus.cc:2781] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:43.470937 25484 raft_consensus.cc:686] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 LEADER]: Becoming Leader. State: Replica: 2ae58cadc2aa4eb79ee907b17ce7a049, State: Running, Role: LEADER
I20240502 02:04:43.470932 24738 raft_consensus.cc:3037] T 4622b7cd6aff4ddeac39b772561004f9 P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:43.471055 25484 consensus_queue.cc:237] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.471892 24738 raft_consensus.cc:2454] T 4622b7cd6aff4ddeac39b772561004f9 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ae58cadc2aa4eb79ee907b17ce7a049 in term 1.
I20240502 02:04:43.472942 25014 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f2ad58ac3a054d62b2ce1729ca6979fa" candidate_uuid: "388f78c5934b4f0caad6ea8aaee10eef" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dfcff51db449bcaeb54a8d97d18942" is_pre_election: true
I20240502 02:04:43.473030 25014 raft_consensus.cc:2454] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 388f78c5934b4f0caad6ea8aaee10eef in term 0.
I20240502 02:04:43.473379 24875 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f2ad58ac3a054d62b2ce1729ca6979fa" candidate_uuid: "388f78c5934b4f0caad6ea8aaee10eef" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" is_pre_election: true
I20240502 02:04:43.473522 24875 raft_consensus.cc:2454] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 388f78c5934b4f0caad6ea8aaee10eef in term 0.
I20240502 02:04:43.473598 24590 catalog_manager.cc:5484] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66). New cstate: current_term: 1 leader_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:43.473848 24669 leader_election.cc:302] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 388f78c5934b4f0caad6ea8aaee10eef, a3dfcff51db449bcaeb54a8d97d18942; no voters:
I20240502 02:04:43.474239 25486 raft_consensus.cc:2781] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 02:04:43.474304 25486 raft_consensus.cc:482] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 02:04:43.474345 25486 raft_consensus.cc:3037] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:43.475203 25486 raft_consensus.cc:504] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.475534 25486 leader_election.cc:288] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [CANDIDATE]: Term 1 election: Requested vote from peers 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66:43575), a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073)
I20240502 02:04:43.475813 24875 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f2ad58ac3a054d62b2ce1729ca6979fa" candidate_uuid: "388f78c5934b4f0caad6ea8aaee10eef" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049"
I20240502 02:04:43.475910 24875 raft_consensus.cc:3037] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:43.475979 25014 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f2ad58ac3a054d62b2ce1729ca6979fa" candidate_uuid: "388f78c5934b4f0caad6ea8aaee10eef" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dfcff51db449bcaeb54a8d97d18942"
I20240502 02:04:43.476064 25014 raft_consensus.cc:3037] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 [term 0 FOLLOWER]: Advancing to term 1
I20240502 02:04:43.476754 24875 raft_consensus.cc:2454] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 388f78c5934b4f0caad6ea8aaee10eef in term 1.
I20240502 02:04:43.476843 25014 raft_consensus.cc:2454] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 388f78c5934b4f0caad6ea8aaee10eef in term 1.
I20240502 02:04:43.477128 24669 leader_election.cc:302] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 388f78c5934b4f0caad6ea8aaee10eef, a3dfcff51db449bcaeb54a8d97d18942; no voters:
I20240502 02:04:43.477312 25486 raft_consensus.cc:2781] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Leader election won for term 1
I20240502 02:04:43.477397 25486 raft_consensus.cc:686] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [term 1 LEADER]: Becoming Leader. State: Replica: 388f78c5934b4f0caad6ea8aaee10eef, State: Running, Role: LEADER
I20240502 02:04:43.477718 25486 consensus_queue.cc:237] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } }
I20240502 02:04:43.479665 24590 catalog_manager.cc:5484] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef reported cstate change: term changed from 0 to 1, leader changed from <none> to 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65). New cstate: current_term: 1 leader_uuid: "388f78c5934b4f0caad6ea8aaee10eef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } health_report { overall_health: UNKNOWN } } }
W20240502 02:04:43.614938 25501 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:43.615146 25501 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:43.617836 25501 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_2'
I20240502 02:04:43.690778 25014 raft_consensus.cc:1261] T e5d1e27205184c7db5678c4fe61c3512 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:43.691226 24738 raft_consensus.cc:1261] T 4622b7cd6aff4ddeac39b772561004f9 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:43.691745 25484 consensus_queue.cc:1035] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:43.691797 25358 consensus_queue.cc:1035] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:43.692561 24737 raft_consensus.cc:1261] T e5d1e27205184c7db5678c4fe61c3512 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:43.693301 25484 consensus_queue.cc:1035] T e5d1e27205184c7db5678c4fe61c3512 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:43.697650 25014 raft_consensus.cc:1261] T 4622b7cd6aff4ddeac39b772561004f9 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:43.699831 24875 raft_consensus.cc:1261] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Refusing update from remote peer 388f78c5934b4f0caad6ea8aaee10eef: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:43.700259 25014 raft_consensus.cc:1261] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Refusing update from remote peer 388f78c5934b4f0caad6ea8aaee10eef: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 02:04:43.700353 25486 consensus_queue.cc:1035] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:43.700368 25479 consensus_queue.cc:1035] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 02:04:43.700708 25360 consensus_queue.cc:1035] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 92.8612 ms
time per row: 0.0464306 ms
I20240502 02:04:43.750897 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.68:38221
--local_ip_for_outbound_sockets=127.7.221.68
--tserver_master_addrs=127.7.221.126:43125
--webserver_port=36071
--webserver_interface=127.7.221.68
--builtin_ntp_servers=127.7.221.84:39799
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 02:04:43.847501 25541 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:43.847697 25541 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:43.847752 25541 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:43.850653 25541 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:43.850709 25541 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:43.850805 25541 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.68
I20240502 02:04:43.854106 25541 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39799
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.68:38221
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.7.221.68
--webserver_port=36071
--tserver_master_addrs=127.7.221.126:43125
--never_fsync=true
--heap_profile_path=/tmp/kudu.25541
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.68
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:43.854854 25541 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:43.855818 25541 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:43.863216 25549 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:43.863984 25541 server_base.cc:1034] running on GCE node
W20240502 02:04:43.864661 25546 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:43.865262 25547 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:43.865633 25541 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:43.866087 25541 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:43.867233 25541 hybrid_clock.cc:648] HybridClock initialized: now 1714615483867207 us; error 36 us; skew 500 ppm
I20240502 02:04:43.869089 25541 webserver.cc:460] Webserver started at http://127.7.221.68:36071/ using document root <none> and password file <none>
I20240502 02:04:43.869570 25541 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:43.869645 25541 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:43.872577 25541 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.001s sys 0.000s
I20240502 02:04:43.874464 25555 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:43.875366 25541 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 02:04:43.875460 25541 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "d6365f5df381444281762b821cfdce85"
format_stamp: "Formatted at 2024-05-02 02:04:38 on dist-test-slave-dmxt"
I20240502 02:04:43.875809 25541 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:43.884840 25541 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:43.885318 25541 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:43.885489 25541 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:43.885953 25541 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:43.886797 25541 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:43.886853 25541 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:43.886919 25541 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:43.886946 25541 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:43.894718 25541 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.68:38221
I20240502 02:04:43.894769 25670 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.68:38221 every 8 connection(s)
I20240502 02:04:43.895639 25541 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 02:04:43.903574 25671 heartbeater.cc:347] Connected to a master server at 127.7.221.126:43125
I20240502 02:04:43.903764 25671 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:43.904254 25671 heartbeater.cc:511] Master 127.7.221.126:43125 requested a full tablet report, sending...
I20240502 02:04:43.905332 24590 ts_manager.cc:194] Re-registered known tserver with Master: d6365f5df381444281762b821cfdce85 (127.7.221.68:38221)
I20240502 02:04:43.905880 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 25541
I20240502 02:04:43.905906 24590 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.68:41049
I20240502 02:04:43.906244 8053 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
/tmp/dist-test-tasko3zIVx/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.7.221.69:43067
--local_ip_for_outbound_sockets=127.7.221.69
--tserver_master_addrs=127.7.221.126:43125
--webserver_port=39651
--webserver_interface=127.7.221.69
--builtin_ntp_servers=127.7.221.84:39799
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 02:04:44.008435 25674 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 02:04:44.008612 25674 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 02:04:44.008652 25674 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 02:04:44.011256 25674 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 02:04:44.011305 25674 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 02:04:44.011382 25674 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.69
I20240502 02:04:44.014467 25674 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.7.221.84:39799
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.7.221.69:43067
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.7.221.69
--webserver_port=39651
--tserver_master_addrs=127.7.221.126:43125
--never_fsync=true
--heap_profile_path=/tmp/kudu.25674
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.7.221.69
--log_dir=/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 01:43:14 UTC on 1a15035a6830
build id 81
I20240502 02:04:44.015164 25674 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 02:04:44.016114 25674 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 02:04:44.023780 25682 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:44.023972 25679 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 02:04:44.024051 25680 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 02:04:44.024751 25674 server_base.cc:1034] running on GCE node
I20240502 02:04:44.025069 25674 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 02:04:44.025528 25674 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 02:04:44.026659 25674 hybrid_clock.cc:648] HybridClock initialized: now 1714615484026635 us; error 43 us; skew 500 ppm
I20240502 02:04:44.028402 25674 webserver.cc:460] Webserver started at http://127.7.221.69:39651/ using document root <none> and password file <none>
I20240502 02:04:44.028901 25674 fs_manager.cc:359] Metadata directory not provided
I20240502 02:04:44.028975 25674 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 02:04:44.031877 25674 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.000s sys 0.003s
I20240502 02:04:44.033573 25688 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:44.034435 25674 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.001s
I20240502 02:04:44.034524 25674 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "7988227e2b74402483fda5bf517f0c0e"
format_stamp: "Formatted at 2024-05-02 02:04:38 on dist-test-slave-dmxt"
I20240502 02:04:44.034916 25674 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 02:04:44.053476 25674 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 02:04:44.053958 25674 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 02:04:44.054091 25674 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 02:04:44.054525 25674 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 02:04:44.055280 25674 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 02:04:44.055342 25674 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:44.055390 25674 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 02:04:44.055423 25674 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 02:04:44.063124 25674 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.69:43067
I20240502 02:04:44.063184 25808 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.221.69:43067 every 8 connection(s)
I20240502 02:04:44.064015 25674 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 02:04:44.071704 25809 heartbeater.cc:347] Connected to a master server at 127.7.221.126:43125
I20240502 02:04:44.071899 25809 heartbeater.cc:464] Registering TS with master...
I20240502 02:04:44.072024 8053 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasko3zIVx/build/debug/bin/kudu as pid 25674
I20240502 02:04:44.072439 25809 heartbeater.cc:511] Master 127.7.221.126:43125 requested a full tablet report, sending...
I20240502 02:04:44.073454 24590 ts_manager.cc:194] Re-registered known tserver with Master: 7988227e2b74402483fda5bf517f0c0e (127.7.221.69:43067)
I20240502 02:04:44.074156 24590 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.69:37897
I20240502 02:04:44.260040 25603 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:44.261437 24716 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:44.262050 24991 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:44.267182 24854 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 02:04:44.273511 25738 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:44.368065 24854 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 02:04:44.371117 24716 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:44.380240 25603 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:44.380806 24991 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:44.389226 25738 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:44.539075 24854 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 02:04:44.541961 24991 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:44.543051 25738 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:44.544226 25603 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:44.547786 24716 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:44.627323 25738 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:44.628952 24991 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 02:04:44.629734 24716 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 02:04:44.631352 25603 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 02:04:44.631778 24854 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 02:04:44.671911 24739 consensus_queue.cc:237] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:44.673487 25014 raft_consensus.cc:1261] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Refusing update from remote peer 388f78c5934b4f0caad6ea8aaee10eef: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:44.673962 24875 raft_consensus.cc:1261] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Refusing update from remote peer 388f78c5934b4f0caad6ea8aaee10eef: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:44.673991 25486 consensus_queue.cc:1035] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:44.674409 25486 consensus_queue.cc:1035] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:44.676020 25962 raft_consensus.cc:2932] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:44.676163 25014 raft_consensus.cc:2932] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:44.678021 24875 raft_consensus.cc:2932] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
W20240502 02:04:44.678717 24670 consensus_peers.cc:485] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef -> Peer d6365f5df381444281762b821cfdce85 (127.7.221.68:38221): Couldn't send request to peer d6365f5df381444281762b821cfdce85. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f2ad58ac3a054d62b2ce1729ca6979fa. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:44.678786 24585 catalog_manager.cc:5484] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 reported cstate change: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "388f78c5934b4f0caad6ea8aaee10eef" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:44.688059 24875 consensus_queue.cc:237] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:44.689574 24739 raft_consensus.cc:1261] T da2526bec2cd41aca7a1210289741c9f P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:44.689625 25014 raft_consensus.cc:1261] T da2526bec2cd41aca7a1210289741c9f P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:44.689953 25975 consensus_queue.cc:1035] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:44.690187 25976 consensus_queue.cc:1035] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:44.692027 25976 raft_consensus.cc:2932] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:44.692397 24739 raft_consensus.cc:2932] T da2526bec2cd41aca7a1210289741c9f P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
W20240502 02:04:44.693498 24808 consensus_peers.cc:485] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 -> Peer d6365f5df381444281762b821cfdce85 (127.7.221.68:38221): Couldn't send request to peer d6365f5df381444281762b821cfdce85. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: da2526bec2cd41aca7a1210289741c9f. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:44.694662 24590 catalog_manager.cc:5484] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049 reported cstate change: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:44.694710 25014 raft_consensus.cc:2932] T da2526bec2cd41aca7a1210289741c9f P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:44.701391 24875 consensus_queue.cc:237] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:44.702584 24739 raft_consensus.cc:1261] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:44.702569 25014 raft_consensus.cc:1261] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:44.702970 25975 consensus_queue.cc:1035] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:44.703123 25980 consensus_queue.cc:1035] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 02:04:44.703296 24808 consensus_peers.cc:485] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 -> Peer d6365f5df381444281762b821cfdce85 (127.7.221.68:38221): Couldn't send request to peer d6365f5df381444281762b821cfdce85. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: dbe35b927da442d19092ca6633a22bb5. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:44.704787 25980 raft_consensus.cc:2932] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:44.705250 25014 raft_consensus.cc:2932] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:44.707371 24584 catalog_manager.cc:5484] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 reported cstate change: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:44.707576 24739 raft_consensus.cc:2932] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:44.714793 25014 consensus_queue.cc:237] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:44.716089 24739 raft_consensus.cc:1261] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer a3dfcff51db449bcaeb54a8d97d18942: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:44.716333 24875 raft_consensus.cc:1261] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Refusing update from remote peer a3dfcff51db449bcaeb54a8d97d18942: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:44.716449 25415 consensus_queue.cc:1035] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:44.716717 25996 consensus_queue.cc:1035] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:44.718253 25415 raft_consensus.cc:2932] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:44.719296 24875 raft_consensus.cc:2932] T db9498139b2f4f73a2e782e6669ee500 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:44.720445 24584 catalog_manager.cc:5484] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 reported cstate change: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "a3dfcff51db449bcaeb54a8d97d18942" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:44.720647 24739 raft_consensus.cc:2932] T db9498139b2f4f73a2e782e6669ee500 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
W20240502 02:04:44.722589 24944 consensus_peers.cc:485] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942 -> Peer d6365f5df381444281762b821cfdce85 (127.7.221.68:38221): Couldn't send request to peer d6365f5df381444281762b821cfdce85. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: db9498139b2f4f73a2e782e6669ee500. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:44.728271 25014 consensus_queue.cc:237] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:44.729522 24875 raft_consensus.cc:1261] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Refusing update from remote peer a3dfcff51db449bcaeb54a8d97d18942: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 02:04:44.729616 24944 consensus_peers.cc:485] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 -> Peer d6365f5df381444281762b821cfdce85 (127.7.221.68:38221): Couldn't send request to peer d6365f5df381444281762b821cfdce85. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: daee40b8ed5446c78a730cd052c9d6b3. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:44.729696 24739 raft_consensus.cc:1261] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer a3dfcff51db449bcaeb54a8d97d18942: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:44.729871 25996 consensus_queue.cc:1035] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:44.730077 26001 consensus_queue.cc:1035] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:44.731307 25996 raft_consensus.cc:2932] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:44.731739 24875 raft_consensus.cc:2932] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:44.732024 24739 raft_consensus.cc:2932] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:44.733428 24590 catalog_manager.cc:5484] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 reported cstate change: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "a3dfcff51db449bcaeb54a8d97d18942" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:44.908118 25671 heartbeater.cc:503] Master 127.7.221.126:43125 was elected leader, sending a full tablet report...
I20240502 02:04:45.076059 25809 heartbeater.cc:503] Master 127.7.221.126:43125 was elected leader, sending a full tablet report...
I20240502 02:04:45.169811 26061 ts_tablet_manager.cc:902] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85: Initiating tablet copy from peer a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073)
I20240502 02:04:45.170476 26061 tablet_copy_client.cc:308] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:43073
I20240502 02:04:45.174189 25034 tablet_copy_service.cc:140] P a3dfcff51db449bcaeb54a8d97d18942: Received BeginTabletCopySession request for tablet daee40b8ed5446c78a730cd052c9d6b3 from peer d6365f5df381444281762b821cfdce85 ({username='slave'} at 127.7.221.68:46311)
I20240502 02:04:45.174314 25034 tablet_copy_service.cc:161] P a3dfcff51db449bcaeb54a8d97d18942: Beginning new tablet copy session on tablet daee40b8ed5446c78a730cd052c9d6b3 from peer d6365f5df381444281762b821cfdce85 at {username='slave'} at 127.7.221.68:46311: session id = d6365f5df381444281762b821cfdce85-daee40b8ed5446c78a730cd052c9d6b3
I20240502 02:04:45.175433 25034 tablet_copy_source_session.cc:205] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:45.178537 26061 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet daee40b8ed5446c78a730cd052c9d6b3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:45.183959 26061 tablet_copy_client.cc:783] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:45.184131 26061 tablet_copy_client.cc:647] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:45.185752 26061 tablet_copy_client.cc:523] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:45.187099 26061 tablet_bootstrap.cc:492] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85: Bootstrap starting.
I20240502 02:04:45.192658 26061 log.cc:824] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85: Log is configured to *not* fsync() on all Append() calls
I20240502 02:04:45.217343 26061 tablet_bootstrap.cc:492] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:45.217669 26061 tablet_bootstrap.cc:492] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85: Bootstrap complete.
I20240502 02:04:45.217975 26061 ts_tablet_manager.cc:1367] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85: Time spent bootstrapping tablet: real 0.031s user 0.028s sys 0.000s
I20240502 02:04:45.220089 26061 raft_consensus.cc:348] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:45.220305 26061 raft_consensus.cc:729] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d6365f5df381444281762b821cfdce85, State: Initialized, Role: LEARNER
I20240502 02:04:45.220595 26061 consensus_queue.cc:260] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:45.221266 26061 ts_tablet_manager.cc:1398] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85: Time spent starting tablet: real 0.003s user 0.006s sys 0.000s
I20240502 02:04:45.221902 25034 tablet_copy_service.cc:342] P a3dfcff51db449bcaeb54a8d97d18942: Request end of tablet copy session d6365f5df381444281762b821cfdce85-daee40b8ed5446c78a730cd052c9d6b3 received from {username='slave'} at 127.7.221.68:46311
I20240502 02:04:45.221988 25034 tablet_copy_service.cc:433] P a3dfcff51db449bcaeb54a8d97d18942: ending tablet copy session d6365f5df381444281762b821cfdce85-daee40b8ed5446c78a730cd052c9d6b3 on tablet daee40b8ed5446c78a730cd052c9d6b3 with peer d6365f5df381444281762b821cfdce85
I20240502 02:04:45.235899 26061 ts_tablet_manager.cc:902] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85: Initiating tablet copy from peer 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66:43575)
I20240502 02:04:45.236259 26061 tablet_copy_client.cc:308] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:43575
I20240502 02:04:45.239418 24892 tablet_copy_service.cc:140] P 2ae58cadc2aa4eb79ee907b17ce7a049: Received BeginTabletCopySession request for tablet dbe35b927da442d19092ca6633a22bb5 from peer d6365f5df381444281762b821cfdce85 ({username='slave'} at 127.7.221.68:44689)
I20240502 02:04:45.239553 24892 tablet_copy_service.cc:161] P 2ae58cadc2aa4eb79ee907b17ce7a049: Beginning new tablet copy session on tablet dbe35b927da442d19092ca6633a22bb5 from peer d6365f5df381444281762b821cfdce85 at {username='slave'} at 127.7.221.68:44689: session id = d6365f5df381444281762b821cfdce85-dbe35b927da442d19092ca6633a22bb5
I20240502 02:04:45.240578 24892 tablet_copy_source_session.cc:205] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:45.241588 26061 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dbe35b927da442d19092ca6633a22bb5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:45.243273 26061 tablet_copy_client.cc:783] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:45.243431 26061 tablet_copy_client.cc:647] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:45.244762 26061 tablet_copy_client.cc:523] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:45.245657 26061 tablet_bootstrap.cc:492] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85: Bootstrap starting.
I20240502 02:04:45.253538 26101 ts_tablet_manager.cc:902] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85: Initiating tablet copy from peer 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793)
I20240502 02:04:45.254079 26101 tablet_copy_client.cc:308] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.65:34793
I20240502 02:04:45.257702 24760 tablet_copy_service.cc:140] P 388f78c5934b4f0caad6ea8aaee10eef: Received BeginTabletCopySession request for tablet f2ad58ac3a054d62b2ce1729ca6979fa from peer d6365f5df381444281762b821cfdce85 ({username='slave'} at 127.7.221.68:52843)
I20240502 02:04:45.257865 24760 tablet_copy_service.cc:161] P 388f78c5934b4f0caad6ea8aaee10eef: Beginning new tablet copy session on tablet f2ad58ac3a054d62b2ce1729ca6979fa from peer d6365f5df381444281762b821cfdce85 at {username='slave'} at 127.7.221.68:52843: session id = d6365f5df381444281762b821cfdce85-f2ad58ac3a054d62b2ce1729ca6979fa
I20240502 02:04:45.258999 24760 tablet_copy_source_session.cc:205] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:45.260031 26101 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f2ad58ac3a054d62b2ce1729ca6979fa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:45.261905 26101 tablet_copy_client.cc:783] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:45.262043 26101 tablet_copy_client.cc:647] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:45.263365 26101 tablet_copy_client.cc:523] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:45.264472 26101 tablet_bootstrap.cc:492] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85: Bootstrap starting.
I20240502 02:04:45.266369 26061 tablet_bootstrap.cc:492] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:45.266584 26061 tablet_bootstrap.cc:492] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85: Bootstrap complete.
I20240502 02:04:45.266780 26061 ts_tablet_manager.cc:1367] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85: Time spent bootstrapping tablet: real 0.021s user 0.022s sys 0.000s
I20240502 02:04:45.267294 26061 raft_consensus.cc:348] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:45.267468 26061 raft_consensus.cc:729] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d6365f5df381444281762b821cfdce85, State: Initialized, Role: LEARNER
I20240502 02:04:45.267570 26061 consensus_queue.cc:260] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:45.267897 26061 ts_tablet_manager.cc:1398] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:45.268448 24892 tablet_copy_service.cc:342] P 2ae58cadc2aa4eb79ee907b17ce7a049: Request end of tablet copy session d6365f5df381444281762b821cfdce85-dbe35b927da442d19092ca6633a22bb5 received from {username='slave'} at 127.7.221.68:44689
I20240502 02:04:45.268546 24892 tablet_copy_service.cc:433] P 2ae58cadc2aa4eb79ee907b17ce7a049: ending tablet copy session d6365f5df381444281762b821cfdce85-dbe35b927da442d19092ca6633a22bb5 on tablet dbe35b927da442d19092ca6633a22bb5 with peer d6365f5df381444281762b821cfdce85
I20240502 02:04:45.270700 26061 ts_tablet_manager.cc:902] T db9498139b2f4f73a2e782e6669ee500 P d6365f5df381444281762b821cfdce85: Initiating tablet copy from peer a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073)
I20240502 02:04:45.271081 26061 tablet_copy_client.cc:308] T db9498139b2f4f73a2e782e6669ee500 P d6365f5df381444281762b821cfdce85: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.67:43073
I20240502 02:04:45.271528 25034 tablet_copy_service.cc:140] P a3dfcff51db449bcaeb54a8d97d18942: Received BeginTabletCopySession request for tablet db9498139b2f4f73a2e782e6669ee500 from peer d6365f5df381444281762b821cfdce85 ({username='slave'} at 127.7.221.68:46311)
I20240502 02:04:45.271634 25034 tablet_copy_service.cc:161] P a3dfcff51db449bcaeb54a8d97d18942: Beginning new tablet copy session on tablet db9498139b2f4f73a2e782e6669ee500 from peer d6365f5df381444281762b821cfdce85 at {username='slave'} at 127.7.221.68:46311: session id = d6365f5df381444281762b821cfdce85-db9498139b2f4f73a2e782e6669ee500
I20240502 02:04:45.272301 25034 tablet_copy_source_session.cc:205] T db9498139b2f4f73a2e782e6669ee500 P a3dfcff51db449bcaeb54a8d97d18942: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:45.272965 26061 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet db9498139b2f4f73a2e782e6669ee500. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:45.274713 26061 tablet_copy_client.cc:783] T db9498139b2f4f73a2e782e6669ee500 P d6365f5df381444281762b821cfdce85: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:45.274945 26061 tablet_copy_client.cc:647] T db9498139b2f4f73a2e782e6669ee500 P d6365f5df381444281762b821cfdce85: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:45.275961 26061 tablet_copy_client.cc:523] T db9498139b2f4f73a2e782e6669ee500 P d6365f5df381444281762b821cfdce85: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:45.276795 26061 tablet_bootstrap.cc:492] T db9498139b2f4f73a2e782e6669ee500 P d6365f5df381444281762b821cfdce85: Bootstrap starting.
I20240502 02:04:45.287495 26101 tablet_bootstrap.cc:492] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:45.287770 26101 tablet_bootstrap.cc:492] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85: Bootstrap complete.
I20240502 02:04:45.287942 26101 ts_tablet_manager.cc:1367] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85: Time spent bootstrapping tablet: real 0.024s user 0.023s sys 0.000s
I20240502 02:04:45.288319 26101 raft_consensus.cc:348] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:45.288436 26101 raft_consensus.cc:729] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d6365f5df381444281762b821cfdce85, State: Initialized, Role: LEARNER
I20240502 02:04:45.288539 26101 consensus_queue.cc:260] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:45.288829 26101 ts_tablet_manager.cc:1398] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 02:04:45.289397 24760 tablet_copy_service.cc:342] P 388f78c5934b4f0caad6ea8aaee10eef: Request end of tablet copy session d6365f5df381444281762b821cfdce85-f2ad58ac3a054d62b2ce1729ca6979fa received from {username='slave'} at 127.7.221.68:52843
I20240502 02:04:45.289494 24760 tablet_copy_service.cc:433] P 388f78c5934b4f0caad6ea8aaee10eef: ending tablet copy session d6365f5df381444281762b821cfdce85-f2ad58ac3a054d62b2ce1729ca6979fa on tablet f2ad58ac3a054d62b2ce1729ca6979fa with peer d6365f5df381444281762b821cfdce85
I20240502 02:04:45.297147 26061 tablet_bootstrap.cc:492] T db9498139b2f4f73a2e782e6669ee500 P d6365f5df381444281762b821cfdce85: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:45.297345 26061 tablet_bootstrap.cc:492] T db9498139b2f4f73a2e782e6669ee500 P d6365f5df381444281762b821cfdce85: Bootstrap complete.
I20240502 02:04:45.297478 26061 ts_tablet_manager.cc:1367] T db9498139b2f4f73a2e782e6669ee500 P d6365f5df381444281762b821cfdce85: Time spent bootstrapping tablet: real 0.021s user 0.018s sys 0.000s
I20240502 02:04:45.297766 26061 raft_consensus.cc:348] T db9498139b2f4f73a2e782e6669ee500 P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:45.297878 26061 raft_consensus.cc:729] T db9498139b2f4f73a2e782e6669ee500 P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d6365f5df381444281762b821cfdce85, State: Initialized, Role: LEARNER
I20240502 02:04:45.297941 26061 consensus_queue.cc:260] T db9498139b2f4f73a2e782e6669ee500 P d6365f5df381444281762b821cfdce85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:45.298229 26061 ts_tablet_manager.cc:1398] T db9498139b2f4f73a2e782e6669ee500 P d6365f5df381444281762b821cfdce85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:45.298771 25034 tablet_copy_service.cc:342] P a3dfcff51db449bcaeb54a8d97d18942: Request end of tablet copy session d6365f5df381444281762b821cfdce85-db9498139b2f4f73a2e782e6669ee500 received from {username='slave'} at 127.7.221.68:46311
I20240502 02:04:45.298849 25034 tablet_copy_service.cc:433] P a3dfcff51db449bcaeb54a8d97d18942: ending tablet copy session d6365f5df381444281762b821cfdce85-db9498139b2f4f73a2e782e6669ee500 on tablet db9498139b2f4f73a2e782e6669ee500 with peer d6365f5df381444281762b821cfdce85
I20240502 02:04:45.310492 26061 ts_tablet_manager.cc:902] T da2526bec2cd41aca7a1210289741c9f P d6365f5df381444281762b821cfdce85: Initiating tablet copy from peer 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66:43575)
I20240502 02:04:45.311071 26061 tablet_copy_client.cc:308] T da2526bec2cd41aca7a1210289741c9f P d6365f5df381444281762b821cfdce85: tablet copy: Beginning tablet copy session from remote peer at address 127.7.221.66:43575
I20240502 02:04:45.311532 24892 tablet_copy_service.cc:140] P 2ae58cadc2aa4eb79ee907b17ce7a049: Received BeginTabletCopySession request for tablet da2526bec2cd41aca7a1210289741c9f from peer d6365f5df381444281762b821cfdce85 ({username='slave'} at 127.7.221.68:44689)
I20240502 02:04:45.311637 24892 tablet_copy_service.cc:161] P 2ae58cadc2aa4eb79ee907b17ce7a049: Beginning new tablet copy session on tablet da2526bec2cd41aca7a1210289741c9f from peer d6365f5df381444281762b821cfdce85 at {username='slave'} at 127.7.221.68:44689: session id = d6365f5df381444281762b821cfdce85-da2526bec2cd41aca7a1210289741c9f
I20240502 02:04:45.312355 24892 tablet_copy_source_session.cc:205] T da2526bec2cd41aca7a1210289741c9f P 2ae58cadc2aa4eb79ee907b17ce7a049: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 02:04:45.313069 26061 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet da2526bec2cd41aca7a1210289741c9f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 02:04:45.314530 26061 tablet_copy_client.cc:783] T da2526bec2cd41aca7a1210289741c9f P d6365f5df381444281762b821cfdce85: tablet copy: Starting download of 0 data blocks...
I20240502 02:04:45.314949 26061 tablet_copy_client.cc:647] T da2526bec2cd41aca7a1210289741c9f P d6365f5df381444281762b821cfdce85: tablet copy: Starting download of 1 WAL segments...
I20240502 02:04:45.316018 26061 tablet_copy_client.cc:523] T da2526bec2cd41aca7a1210289741c9f P d6365f5df381444281762b821cfdce85: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 02:04:45.316988 26061 tablet_bootstrap.cc:492] T da2526bec2cd41aca7a1210289741c9f P d6365f5df381444281762b821cfdce85: Bootstrap starting.
I20240502 02:04:45.331432 26061 tablet_bootstrap.cc:492] T da2526bec2cd41aca7a1210289741c9f P d6365f5df381444281762b821cfdce85: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 02:04:45.331665 26061 tablet_bootstrap.cc:492] T da2526bec2cd41aca7a1210289741c9f P d6365f5df381444281762b821cfdce85: Bootstrap complete.
I20240502 02:04:45.331825 26061 ts_tablet_manager.cc:1367] T da2526bec2cd41aca7a1210289741c9f P d6365f5df381444281762b821cfdce85: Time spent bootstrapping tablet: real 0.015s user 0.014s sys 0.000s
I20240502 02:04:45.332139 26061 raft_consensus.cc:348] T da2526bec2cd41aca7a1210289741c9f P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:45.332247 26061 raft_consensus.cc:729] T da2526bec2cd41aca7a1210289741c9f P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d6365f5df381444281762b821cfdce85, State: Initialized, Role: LEARNER
I20240502 02:04:45.332357 26061 consensus_queue.cc:260] T da2526bec2cd41aca7a1210289741c9f P d6365f5df381444281762b821cfdce85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:45.332587 26061 ts_tablet_manager.cc:1398] T da2526bec2cd41aca7a1210289741c9f P d6365f5df381444281762b821cfdce85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 02:04:45.333031 24892 tablet_copy_service.cc:342] P 2ae58cadc2aa4eb79ee907b17ce7a049: Request end of tablet copy session d6365f5df381444281762b821cfdce85-da2526bec2cd41aca7a1210289741c9f received from {username='slave'} at 127.7.221.68:44689
I20240502 02:04:45.333099 24892 tablet_copy_service.cc:433] P 2ae58cadc2aa4eb79ee907b17ce7a049: ending tablet copy session d6365f5df381444281762b821cfdce85-da2526bec2cd41aca7a1210289741c9f on tablet da2526bec2cd41aca7a1210289741c9f with peer d6365f5df381444281762b821cfdce85
I20240502 02:04:45.646795 25625 raft_consensus.cc:1203] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 02:04:45.729717 25625 raft_consensus.cc:1203] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 02:04:45.793092 25625 raft_consensus.cc:1203] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 02:04:45.845217 25625 raft_consensus.cc:1203] T da2526bec2cd41aca7a1210289741c9f P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 02:04:45.869458 25625 raft_consensus.cc:1203] T db9498139b2f4f73a2e782e6669ee500 P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 02:04:46.125386 25997 raft_consensus.cc:1050] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942: attempting to promote NON_VOTER d6365f5df381444281762b821cfdce85 to VOTER
I20240502 02:04:46.125845 25997 consensus_queue.cc:237] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } }
I20240502 02:04:46.127070 24739 raft_consensus.cc:1261] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer a3dfcff51db449bcaeb54a8d97d18942: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:46.127142 24875 raft_consensus.cc:1261] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Refusing update from remote peer a3dfcff51db449bcaeb54a8d97d18942: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:46.127300 25625 raft_consensus.cc:1261] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Refusing update from remote peer a3dfcff51db449bcaeb54a8d97d18942: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:46.127445 26142 consensus_queue.cc:1035] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:46.127758 26227 consensus_queue.cc:1035] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:46.128019 26142 consensus_queue.cc:1035] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:46.129835 26227 raft_consensus.cc:2932] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d6365f5df381444281762b821cfdce85 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.130278 24875 raft_consensus.cc:2932] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d6365f5df381444281762b821cfdce85 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.131938 24739 raft_consensus.cc:2932] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d6365f5df381444281762b821cfdce85 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.132740 24585 catalog_manager.cc:5484] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 reported cstate change: config changed from index 4 to 5, d6365f5df381444281762b821cfdce85 (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "a3dfcff51db449bcaeb54a8d97d18942" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 02:04:46.133770 25625 raft_consensus.cc:2932] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d6365f5df381444281762b821cfdce85 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } attrs { replace: true } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.137606 25014 consensus_queue.cc:237] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } }
I20240502 02:04:46.138715 25625 raft_consensus.cc:1261] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85 [term 1 FOLLOWER]: Refusing update from remote peer a3dfcff51db449bcaeb54a8d97d18942: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:46.138833 24875 raft_consensus.cc:1261] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Refusing update from remote peer a3dfcff51db449bcaeb54a8d97d18942: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:46.139101 26143 consensus_queue.cc:1035] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:46.139261 26142 consensus_queue.cc:1035] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:46.140587 25997 raft_consensus.cc:2932] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.140978 24875 raft_consensus.cc:2932] T daee40b8ed5446c78a730cd052c9d6b3 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.141088 25625 raft_consensus.cc:2932] T daee40b8ed5446c78a730cd052c9d6b3 P d6365f5df381444281762b821cfdce85 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.142038 24576 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet daee40b8ed5446c78a730cd052c9d6b3 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:46.142985 24592 catalog_manager.cc:5484] T daee40b8ed5446c78a730cd052c9d6b3 P a3dfcff51db449bcaeb54a8d97d18942 reported cstate change: config changed from index 5 to 6, VOTER 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65) evicted. New cstate: current_term: 1 leader_uuid: "a3dfcff51db449bcaeb54a8d97d18942" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:46.145444 24716 tablet_service.cc:1514] Processing DeleteTablet for tablet daee40b8ed5446c78a730cd052c9d6b3 with delete_type TABLET_DATA_TOMBSTONED (TS 388f78c5934b4f0caad6ea8aaee10eef not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:59988
I20240502 02:04:46.146011 26235 tablet_replica.cc:330] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef: stopping tablet replica
I20240502 02:04:46.146311 26235 raft_consensus.cc:2229] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:46.146521 26235 raft_consensus.cc:2258] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:46.147572 26235 ts_tablet_manager.cc:1875] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:46.149641 25962 raft_consensus.cc:1050] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef: attempting to promote NON_VOTER d6365f5df381444281762b821cfdce85 to VOTER
I20240502 02:04:46.149861 26235 ts_tablet_manager.cc:1888] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 02:04:46.149936 26235 log.cc:1196] T daee40b8ed5446c78a730cd052c9d6b3 P 388f78c5934b4f0caad6ea8aaee10eef: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-0/wal/wals/daee40b8ed5446c78a730cd052c9d6b3
I20240502 02:04:46.150032 25962 consensus_queue.cc:237] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } }
I20240502 02:04:46.150414 24579 catalog_manager.cc:4830] TS 388f78c5934b4f0caad6ea8aaee10eef (127.7.221.65:34793): tablet daee40b8ed5446c78a730cd052c9d6b3 (table rebalance_test_table_1 [id=82ee71ea745f4ac2954d9d8005e40320]) successfully deleted
I20240502 02:04:46.151252 25625 raft_consensus.cc:1261] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Refusing update from remote peer 388f78c5934b4f0caad6ea8aaee10eef: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:46.151252 25014 raft_consensus.cc:1261] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Refusing update from remote peer 388f78c5934b4f0caad6ea8aaee10eef: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:46.151350 24875 raft_consensus.cc:1261] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Refusing update from remote peer 388f78c5934b4f0caad6ea8aaee10eef: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:46.151593 26237 consensus_queue.cc:1035] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:46.151767 25962 consensus_queue.cc:1035] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [LEADER]: Connected to new peer: Peer: permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:46.151921 26171 consensus_queue.cc:1035] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:46.153853 26171 raft_consensus.cc:2932] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d6365f5df381444281762b821cfdce85 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.154336 25014 raft_consensus.cc:2932] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d6365f5df381444281762b821cfdce85 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.156306 24875 raft_consensus.cc:2932] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d6365f5df381444281762b821cfdce85 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.156309 25625 raft_consensus.cc:2932] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d6365f5df381444281762b821cfdce85 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.156623 24590 catalog_manager.cc:5484] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 reported cstate change: config changed from index 4 to 5, d6365f5df381444281762b821cfdce85 (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "388f78c5934b4f0caad6ea8aaee10eef" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } attrs { replace: true } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.160241 24739 consensus_queue.cc:237] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } }
I20240502 02:04:46.161270 25014 raft_consensus.cc:1261] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Refusing update from remote peer 388f78c5934b4f0caad6ea8aaee10eef: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:46.161284 25625 raft_consensus.cc:1261] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85 [term 1 FOLLOWER]: Refusing update from remote peer 388f78c5934b4f0caad6ea8aaee10eef: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:46.161590 25962 consensus_queue.cc:1035] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:46.161746 26237 consensus_queue.cc:1035] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [LEADER]: Connected to new peer: Peer: permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:46.163379 26172 raft_consensus.cc:2932] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.163906 25625 raft_consensus.cc:2932] T f2ad58ac3a054d62b2ce1729ca6979fa P d6365f5df381444281762b821cfdce85 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.164885 24579 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f2ad58ac3a054d62b2ce1729ca6979fa with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:46.165187 25014 raft_consensus.cc:2932] T f2ad58ac3a054d62b2ce1729ca6979fa P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.165750 24585 catalog_manager.cc:5484] T f2ad58ac3a054d62b2ce1729ca6979fa P 388f78c5934b4f0caad6ea8aaee10eef reported cstate change: config changed from index 5 to 6, VOTER 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66) evicted. New cstate: current_term: 1 leader_uuid: "388f78c5934b4f0caad6ea8aaee10eef" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:46.168392 24854 tablet_service.cc:1514] Processing DeleteTablet for tablet f2ad58ac3a054d62b2ce1729ca6979fa with delete_type TABLET_DATA_TOMBSTONED (TS 2ae58cadc2aa4eb79ee907b17ce7a049 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:34622
I20240502 02:04:46.170756 26241 tablet_replica.cc:330] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049: stopping tablet replica
I20240502 02:04:46.171044 26241 raft_consensus.cc:2229] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:46.171248 26241 raft_consensus.cc:2258] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:46.172241 26241 ts_tablet_manager.cc:1875] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:46.174007 26241 ts_tablet_manager.cc:1888] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 02:04:46.174088 26241 log.cc:1196] T f2ad58ac3a054d62b2ce1729ca6979fa P 2ae58cadc2aa4eb79ee907b17ce7a049: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-1/wal/wals/f2ad58ac3a054d62b2ce1729ca6979fa
I20240502 02:04:46.174646 24577 catalog_manager.cc:4830] TS 2ae58cadc2aa4eb79ee907b17ce7a049 (127.7.221.66:43575): tablet f2ad58ac3a054d62b2ce1729ca6979fa (table rebalance_test_table_2 [id=b948bf193f204440a46af2aac2295c6e]) successfully deleted
I20240502 02:04:46.184000 26173 raft_consensus.cc:1050] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049: attempting to promote NON_VOTER d6365f5df381444281762b821cfdce85 to VOTER
I20240502 02:04:46.184446 26173 consensus_queue.cc:237] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } }
I20240502 02:04:46.185770 25625 raft_consensus.cc:1261] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85 [term 1 LEARNER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:46.185807 25014 raft_consensus.cc:1261] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:46.185812 24739 raft_consensus.cc:1261] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 02:04:46.186211 26243 consensus_queue.cc:1035] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:46.186410 26173 consensus_queue.cc:1035] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:46.186553 25975 consensus_queue.cc:1035] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 02:04:46.188488 26173 raft_consensus.cc:2932] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d6365f5df381444281762b821cfdce85 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.188910 24739 raft_consensus.cc:2932] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d6365f5df381444281762b821cfdce85 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.189437 25014 raft_consensus.cc:2932] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d6365f5df381444281762b821cfdce85 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.190665 25625 raft_consensus.cc:2932] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, d6365f5df381444281762b821cfdce85 (127.7.221.68) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.191661 24585 catalog_manager.cc:5484] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 reported cstate change: config changed from index 4 to 5, d6365f5df381444281762b821cfdce85 (127.7.221.68) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 02:04:46.194715 24875 consensus_queue.cc:237] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } }
I20240502 02:04:46.195801 24739 raft_consensus.cc:1261] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:46.195914 25625 raft_consensus.cc:1261] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85 [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 02:04:46.196313 25975 consensus_queue.cc:1035] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:46.196463 26243 consensus_queue.cc:1035] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 02:04:46.198002 26174 raft_consensus.cc:2932] T dbe35b927da442d19092ca6633a22bb5 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.198365 24739 raft_consensus.cc:2932] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.198365 25625 raft_consensus.cc:2932] T dbe35b927da442d19092ca6633a22bb5 P d6365f5df381444281762b821cfdce85 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.199482 24577 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet dbe35b927da442d19092ca6633a22bb5 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 02:04:46.200621 24592 catalog_manager.cc:5484] T dbe35b927da442d19092ca6633a22bb5 P 388f78c5934b4f0caad6ea8aaee10eef reported cstate change: config changed from index 5 to 6, VOTER a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67) evicted. New cstate: current_term: 1 leader_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: false } } }
I20240502 02:04:46.202680 24991 tablet_service.cc:1514] Processing DeleteTablet for tablet dbe35b927da442d19092ca6633a22bb5 with delete_type TABLET_DATA_TOMBSTONED (TS a3dfcff51db449bcaeb54a8d97d18942 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:47522
I20240502 02:04:46.203320 26247 tablet_replica.cc:330] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942: stopping tablet replica
I20240502 02:04:46.203660 26247 raft_consensus.cc:2229] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 02:04:46.203882 26247 raft_consensus.cc:2258] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 02:04:46.204911 26247 ts_tablet_manager.cc:1875] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 02:04:46.206677 26247 ts_tablet_manager.cc:1888] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 02:04:46.206754 26247 log.cc:1196] T dbe35b927da442d19092ca6633a22bb5 P a3dfcff51db449bcaeb54a8d97d18942: Deleting WAL directory at /tmp/dist-test-tasko3zIVx/test-tmp/rebalancer_tool-test.3.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx2_rf3_t3_l_3_2.1714615092046232-8053-0/raft_consensus-itest-cluster/ts-2/wal/wals/dbe35b927da442d19092ca6633a22bb5
I20240502 02:04:46.207273 24576 catalog_manager.cc:4830] TS a3dfcff51db449bcaeb54a8d97d18942 (127.7.221.67:43073): tablet dbe35b927da442d19092ca6633a22bb5 (table rebalance_test_table_0 [id=764f416bfad747f4b2b400623dbbfe73]) successfully deleted
I20240502 02:04:46.360064 24875 consensus_queue.cc:237] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } }
I20240502 02:04:46.361394 24739 raft_consensus.cc:1261] T 4622b7cd6aff4ddeac39b772561004f9 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 02:04:46.361394 25014 raft_consensus.cc:1261] T 4622b7cd6aff4ddeac39b772561004f9 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Refusing update from remote peer 2ae58cadc2aa4eb79ee907b17ce7a049: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 02:04:46.361541 24808 consensus_peers.cc:485] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 -> Peer d6365f5df381444281762b821cfdce85 (127.7.221.68:38221): Couldn't send request to peer d6365f5df381444281762b821cfdce85. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4622b7cd6aff4ddeac39b772561004f9. This is attempt 1: this message will repeat every 5th retry.
I20240502 02:04:46.361814 25975 consensus_queue.cc:1035] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:46.361964 26174 consensus_queue.cc:1035] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [LEADER]: Connected to new peer: Peer: permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 02:04:46.363649 26174 raft_consensus.cc:2932] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:46.364084 25014 raft_consensus.cc:2932] T 4622b7cd6aff4ddeac39b772561004f9 P a3dfcff51db449bcaeb54a8d97d18942 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:46.364307 24739 raft_consensus.cc:2932] T 4622b7cd6aff4ddeac39b772561004f9 P 388f78c5934b4f0caad6ea8aaee10eef [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68" port: 38221 } attrs { promote: true } } }
I20240502 02:04:46.366365 24590 catalog_manager.cc:5484] T 4622b7cd6aff4ddeac39b772561004f9 P 2ae58cadc2aa4eb79ee907b17ce7a049 reported cstate change: config changed from index -1 to 4, NON_VOTER d6365f5df381444281762b821cfdce85 (127.7.221.68) added. New cstate: current_term: 1 leader_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "388f78c5934b4f0caad6ea8aaee10eef" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34793 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2ae58cadc2aa4eb79ee907b17ce7a049" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 43575 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a3dfcff51db449bcaeb54a8d97d18942" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 43073 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d6365f5df381444281762b821cfdce85" member_type: NON_VOTER last_known_addr { host: "127.7.221.68"